Skip to content

MiniProject2 Phase 1 Git Links #43

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

MiniProject2 Phase 1 Git Links #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d52b4b3
Added MiniProject2_Part1
adesai07 Oct 29, 2018
8558ca7
Renamed files
adesai07 Nov 1, 2018
18bcdb2
Deleted old files
adesai07 Nov 1, 2018
6c54d49
Add files via upload
adesai07 Nov 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
278 changes: 278 additions & 0 deletions MiniProject2_Part1.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,278 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 70,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"100 documents deleted.\n",
"Done with getgitlab\n",
"50\n",
"Done with get sourceforge\n",
"50\n"
]
}
],
"source": [
"import sys\n",
"import re\n",
"import pymongo\n",
"import json\n",
"import time\n",
"import datetime\n",
"import requests\n",
"from bs4 import BeautifulSoup\n",
"\n",
"dbname = \"fdac18mp2\" #please use this database\n",
"collname = \"glprj_adesai6\" #please modify so you store data in your collection\n",
"# beginning page index\n",
"begin = \"0\"\n",
"client = pymongo.MongoClient()\n",
"\n",
"db = client[dbname]\n",
"coll = db[collname]\n",
"mchar = 'm'\n",
"\n",
"beginurl = \"https://gitlab.com/api/v4/projects?archived=false&membership=false&order_by=created_at&owned=false&page=\" + begin + \\\n",
" \"&per_page=99&simple=false&sort=desc&starred=false&statistics=false&with_custom_attributes=false&with_issues_enabled=false&with_merge_requests_enabled=false\"\n",
"\n",
"source = \"https://sourceforge.net/directory/?q=\" + mchar + \"&sort=name&page=\"\n",
"rest = \"https://sourceforge.net/rest/p/\"\n",
"\n",
"gleft = 20\n",
"\n",
"header = {'per_page': '99'}\n",
"\n",
"gitlaburls = []\n",
"sourceurls = []\n",
"hasgit = False\n",
"\n",
"\n",
"\n",
"# check remaining query chances for rate-limit restriction\n",
"def wait(left):\n",
" global header\n",
" while (left < 20):\n",
" l = requests.get('https://gitlab.com/api/v4/projects', headers=header)\n",
" if (l.ok):\n",
" left = int(l.headers.get('RateLimit-Remaining'))\n",
" time .sleep(60)\n",
" return left\n",
"\n",
"def projectreturns(url):\n",
" r = requests.get(url)\n",
" if r.status_code == 200:\n",
" return True\n",
" return False\n",
"\n",
"def getsourceforge():\n",
" page = 1\n",
" projectnum = 0\n",
" \n",
" file = open('sourceforgelinks.txt', 'w') \n",
"\n",
" while True:\n",
" resp = requests.get(source + str(page))\n",
" text = resp.text\n",
" soup = BeautifulSoup(text, 'html.parser')\n",
" if re.search('No results found.', soup.get_text()):\n",
" return\n",
"\n",
" for link in soup.find_all(class_=\"project-icon\", href=True):\n",
" hasgit = False \n",
" if projectnum >= 50:\n",
" file.close()\n",
" return\n",
" name = re.findall('/projects/([A-Za-z0-9\\-]*)', link.get('href'))\n",
" name = name[0] if name else None\n",
" if name is not None and name.lower().startswith(mchar):\n",
" resp = requests.get(rest + name)\n",
" if resp.status_code == 200:\n",
" info = json.loads(resp.text) \n",
" for x in range(0, len(info['tools'])):\n",
" if info['tools'][x]['name'] == 'git':\n",
" hasgit = True\n",
" break\n",
" info['forge'] = 'sourceforge'\n",
" fullurl = info['url'] + 'code/'\n",
" #print(fullurl)\n",
" if hasgit == True:\n",
" if projectreturns(fullurl):\n",
" sourceurls.append(fullurl)\n",
" file.write(fullurl)\n",
" file.write('\\n')\n",
" existing = coll.find_one(info)\n",
" if existing is None:\n",
" coll.insert_one(info)\n",
" else:\n",
" continue\n",
" projectnum += 1\n",
" \n",
" \n",
" page += 1\n",
" return\n",
"\n",
"def getgitlab():\n",
"\n",
" global gleft\n",
" global header\n",
" global bginnum\n",
" gleft = wait(gleft)\n",
" values = []\n",
" size = 0\n",
" project_count = 0\n",
"\n",
" try:\n",
" r = requests .get(beginurl, headers=header)\n",
" time .sleep(0.5)\n",
" # got blocked\n",
" if r.status_code == 403:\n",
" return \"got blocked\", str(bginnum)\n",
" if (r.ok):\n",
"\n",
" gleft = int(r.headers.get('RateLimit-Remaining'))\n",
" lll = r.headers.get('Link')\n",
" t = r.text\n",
" array = json.loads(t)\n",
" \n",
" for el in array:\n",
" if project_count >= 50:\n",
" return\n",
" if el['name'].lower().startswith(mchar):\n",
" if projectreturns(el['http_url_to_repo']):\n",
" project_count += 1\n",
" el['forge'] = 'gitlab'\n",
" coll.insert_one(el)\n",
" gitlaburls.append(el['http_url_to_repo'])\n",
" \n",
" \n",
" #next page\n",
" while ('; rel=\"next\"' in lll):\n",
" gleft = int(r.headers.get('RateLimit-Remaining'))\n",
" gleft = wait(gleft)\n",
" # extract next page url\n",
" ll = lll.replace(';', ',').split(',')\n",
" url = ll[ll.index(' rel=\"next\"') -\n",
" 1].replace('<', '').replace('>', '').lstrip()\n",
" \n",
" try:\n",
" r = requests .get(url, headers=header)\n",
" if r.status_code == 403:\n",
" return \"got blocked\", str(bginnum)\n",
" if (r.ok):\n",
" lll = r.headers.get('Link')\n",
" t = r.text\n",
" array1 = json.loads(t)\n",
" for el in array1:\n",
" if project_count >= 50:\n",
" return\n",
" if el['name'].lower().startswith(mchar):\n",
" if projectreturns(el['http_url_to_repo']):\n",
" project_count += 1\n",
" el['forge'] = 'gitlab'\n",
" coll.insert_one(el)\n",
" gitlaburls.append(el['http_url_to_repo'])\n",
" \n",
" else:\n",
" sys.stderr.write(\"url can not found:\\n\" + url + '\\n')\n",
" return \n",
" except requests.exceptions.ConnectionError:\n",
" sys.stderr.write('could not get ' + url + '\\n')\n",
"\n",
" else:\n",
" sys.stderr.write(\"url can not found:\\n\" + url + '\\n')\n",
" return\n",
"\n",
" except requests.exceptions.ConnectionError:\n",
" sys.stderr.write('could not get ' + url + '\\n')\n",
" except Exception as e:\n",
" sys.stderr.write(url + ';' + str(e) + '\\n')\n",
" \n",
"x = coll.delete_many({})\n",
"print(x.deleted_count, \" documents deleted.\")\n",
"\n",
"getgitlab()\n",
"print(\"Done with getgitlab\")\n",
"print(len(gitlaburls))\n",
"getsourceforge()\n",
"print(\"Done with get sourceforge\") \n",
"print(len(sourceurls))\n",
"\n",
"links = open('gitLinks.txt', 'a')\n",
"links.write('\\n')\n",
" \n",
"for x in gitlaburls:\n",
" links.write(x)\n",
" links.write('\\n')\n",
"\n",
"links.close()\n",
" "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
2 changes: 1 addition & 1 deletion compareRels.py → adesai6_compareRels.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
headers = {'Accept': 'application/vnd.github.hellcat-preview+json'}

db = client['fdac18mp2'] # added in class
collName = 'releases_audris'
collName = 'releases_adesai6'
coll = db [collName]
def wait (left):
while (left < 20):
Expand Down
2 changes: 1 addition & 1 deletion extrNpm.py → adesai6_extrNpm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import pymongo, json, sys
client = pymongo.MongoClient (host="da1")
db = client ['fdac18mp2']
id = "audris"
id = "adesai6"
coll = db [ 'npm_' + id]
for r in coll.find():
if 'collected' in r:
Expand Down
2 changes: 1 addition & 1 deletion extrRels.py → adesai6_extrRels.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import pymongo, json, sys
client = pymongo.MongoClient (host="da1")
db = client ['fdac18mp2']
id = "audris"
id = "adesai6"
coll = db [ 'releases_' + id]
for r in coll.find():
n = r['name']
Expand Down
Loading