Update RepoPipeline.py
Browse files- RepoPipeline.py +3 -2
RepoPipeline.py
CHANGED
@@ -48,7 +48,7 @@ def extract_requirements(lines):
|
|
48 |
splitLine = line.split(" == ")
|
49 |
else:
|
50 |
splitLine = line.split("==")
|
51 |
-
requirements_set.
|
52 |
except:
|
53 |
pass
|
54 |
return requirements_set
|
@@ -131,7 +131,8 @@ def extract_information(repos, headers=None):
|
|
131 |
try:
|
132 |
file_content = tar.extractfile(member).read().decode("utf-8")
|
133 |
# extract readme
|
134 |
-
readmes_set = set(
|
|
|
135 |
repo_info["readmes"].update(readmes_set)
|
136 |
except UnicodeDecodeError as e:
|
137 |
tqdm.write(
|
|
|
48 |
splitLine = line.split(" == ")
|
49 |
else:
|
50 |
splitLine = line.split("==")
|
51 |
+
requirements_set.add(splitLine[0])
|
52 |
except:
|
53 |
pass
|
54 |
return requirements_set
|
|
|
131 |
try:
|
132 |
file_content = tar.extractfile(member).read().decode("utf-8")
|
133 |
# extract readme
|
134 |
+
readmes_set = set()
|
135 |
+
readmes_set.add(file_content)
|
136 |
repo_info["readmes"].update(readmes_set)
|
137 |
except UnicodeDecodeError as e:
|
138 |
tqdm.write(
|