code ready to push to master

.ipynb_checkpoints/eherron5-checkpoint.ipynb

@@ -0,0 +1,327 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 139,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import libraries\n",
+    "import sys\n",
+    "import re\n",
+    "import pymongo\n",
+    "import json\n",
+    "import time\n",
+    "import datetime\n",
+    "import requests\n",
+    "from bs4 import BeautifulSoup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 140,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# specifications\n",
+    "dbname = \"fdac18mp2\" #please use this database\n",
+    "gl_collname = \"glprj_eherron5\"\n",
+    "sf_collname = \"sfprj_eherron5\"\n",
+    "my_char = 'o'\n",
+    "\n",
+    "# beginning page index\n",
+    "begin = \"0\"\n",
+    "client = pymongo.MongoClient()\n",
+    "\n",
+    "db = client[dbname]\n",
+    "gl_coll = db[gl_collname]\n",
+    "sf_coll = db[sf_collname]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 141,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# urls\n",
+    "gl_url = \"https://gitlab.com/api/v4/projects?archived=false&membership=false&order_by=created_at&owned=false&page=\" + begin + \\\n",
+    "    \"&per_page=99&simple=false&sort=desc&starred=false&statistics=false&with_custom_attributes=false&with_issues_enabled=false&with_merge_requests_enabled=false\"\n",
+    "sf_url = \"https://sourceforge.net/directory/?q=\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 142,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gleft = 0\n",
+    "\n",
+    "header = {'per_page': 99}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 143,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def url_exists(url):\n",
+    "    request = requests.get(url)\n",
+    "    return request.status_code == 200"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 127,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# check remaining query chances for rate-limit restriction\n",
+    "def gl_wait(left):\n",
+    "    global header\n",
+    "    while (left < 20):\n",
+    "        l = requests.get('https://gitlab.com/api/v4/projects', headers=header)\n",
+    "        if (l.ok):\n",
+    "            left = int(l.headers.get('RateLimit-Remaining'))\n",
+    "        time .sleep(60)\n",
+    "    return left"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 128,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# send queries and extract urls - gitlab\n",
+    "def gl_get(url, coll, start_char):\n",
+    "\n",
+    "    global gleft\n",
+    "    global header\n",
+    "    global bginnum\n",
+    "    gleft = gl_wait(gleft)\n",
+    "    values = []\n",
+    "    size = 0\n",
+    "\n",
+    "    max_links = 50\n",
+    "    links_count = 0\n",
+    "    \n",
+    "    try:\n",
+    "        r = requests .get(url, headers=header)\n",
+    "        time .sleep(0.5)\n",
+    "        # got blocked\n",
+    "        if r.status_code == 403:\n",
+    "            return \"got blocked\", str(bginnum)\n",
+    "        if (r.ok):\n",
+    "\n",
+    "            gleft = int(r.headers.get('RateLimit-Remaining'))\n",
+    "            lll = r.headers.get('Link')\n",
+    "            t = r.text\n",
+    "            array = json.loads(t)\n",
+    "\n",
+    "            for el in array:\n",
+    "                proj_name = el['path'].lower()\n",
+    "                proj_url = el['http_url_to_repo']\n",
+    "                if proj_name.startswith(start_char) and url_exists(proj_url):\n",
+    "                    print('inserting url', links_count, ' for path', el['path'], proj_url)\n",
+    "                    coll.insert(el)\n",
+    "                    links_count += 1\n",
+    "                    if links_count >= max_links:\n",
+    "                        return\n",
+    "\n",
+    "            #next page\n",
+    "            while ('; rel=\"next\"' in lll):\n",
+    "                gleft = int(r.headers.get('RateLimit-Remaining'))\n",
+    "                gleft = gl_wait(gleft)\n",
+    "                # extract next page url\n",
+    "                ll = lll.replace(';', ',').split(',')\n",
+    "                url = ll[ll.index(' rel=\"next\"') -\n",
+    "                         1].replace('<', '').replace('>', '').lstrip()\n",
+    "\n",
+    "                try:\n",
+    "                    r = requests .get(url, headers=header)\n",
+    "                    if r.status_code == 403:\n",
+    "                        return \"got blocked\", str(bginnum)\n",
+    "                    if (r.ok):\n",
+    "                        lll = r.headers.get('Link')\n",
+    "                        t = r.text\n",
+    "                        array1 = json.loads(t)\n",
+    "                        for el in array1:\n",
+    "                            coll.insert(el)\n",
+    "                    else:\n",
+    "                        sys.stderr.write(\"url can not found:\\n\" + url + '\\n')\n",
+    "                        return\n",
+    "                except requests.exceptions.ConnectionError:\n",
+    "                    sys.stderr.write('could not get ' + url + '\\n')\n",
+    "\n",
+    "        else:\n",
+    "            sys.stderr.write(\"url can not found:\\n\" + url + '\\n')\n",
+    "            return\n",
+    "\n",
+    "    except requests.exceptions.ConnectionError:\n",
+    "        sys.stderr.write('could not get ' + url + '\\n')\n",
+    "    except Exception as e:\n",
+    "        sys.stderr.write(url + ';' + str(e) + '\\n')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "inserting url 0  for path openmw https://gitlab.com/Fynjyfun/openmw.git\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:32: DeprecationWarning: insert is deprecated. Use insert_one or insert_many instead.\n",
+      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:55: DeprecationWarning: insert is deprecated. Use insert_one or insert_many instead.\n"
+     ]
+    }
+   ],
+   "source": [
+    "#start retrieving gitlab\n",
+    "gl_get(gl_url,gl_coll, my_char)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 144,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def sf_get(url, coll, start_char):\n",
+    "    links_count = 0\n",
+    "    max_links = 50\n",
+    "    page_num = 1\n",
+    "    rest = \"http://sourceforge.net/rest/p/\"\n",
+    "    \n",
+    "    while url_exists(url+ str(page_num)):\n",
+    "        r = requests.get(url+ str(page_num))\n",
+    "        \n",
+    "        # gets html text\n",
+    "        text = r.text\n",
+    "        soup = BeautifulSoup(text, 'html.parser')\n",
+    "        \n",
+    "        # find all projects listed on page\n",
+    "        for item in soup.find_all(class_=\"result-heading-texts\"):\n",
+    "            \n",
+    "            a = item.find('a')\n",
+    "            link = a['href']\n",
+    "            name = link.split('/')[1]\n",
+    "            title = a.get_text()\n",
+    "\n",
+    "            if title.lower().startswith(start_char) and url_exists(rest + name):\n",
+    "                coll.insert_one(requests.get(rest + name).json())\n",
+    "                links_count += 1\n",
+    "                if links_count >= max_links:\n",
+    "                    return\n",
+    "        page_num += 1\n",
+    "    return\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 145,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1\n",
+      "2\n",
+      "3\n",
+      "4\n",
+      "5\n",
+      "6\n",
+      "7\n",
+      "8\n",
+      "9\n",
+      "10\n",
+      "11\n",
+      "12\n",
+      "13\n",
+      "14\n",
+      "15\n",
+      "16\n",
+      "17\n",
+      "18\n",
+      "19\n",
+      "20\n",
+      "21\n",
+      "22\n",
+      "23\n",
+      "24\n",
+      "25\n",
+      "26\n",
+      "27\n",
+      "28\n",
+      "29\n",
+      "30\n",
+      "31\n",
+      "32\n",
+      "33\n",
+      "34\n",
+      "35\n",
+      "36\n",
+      "37\n",
+      "38\n",
+      "39\n",
+      "40\n",
+      "41\n",
+      "42\n",
+      "43\n",
+      "44\n",
+      "45\n",
+      "46\n",
+      "47\n",
+      "48\n",
+      "49\n",
+      "50\n"
+     ]
+    }
+   ],
+   "source": [
+    "# get 50 source forge projects\n",
+    "sf_get(sf_url, sf_coll, my_char)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.5.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

connect.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+#change PREFIX to fdac-yourid
+PREFIX=fdac18-eherron5
+docker-machine start $PREFIX-1
+IP=$(docker-machine ip $PREFIX-1)
+docker-machine ssh $PREFIX-1 "sudo docker start $PREFIX"
+ssh -p443 -i id_rsa_gcloud -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null \
+ -L8889:localhost:8888 \
+ -R27017:da1.eecs.utk.edu:27017 \
+  jovyan@$IP
+