Dockerhub, Appsody releases, and star/watcher/forks for /stacks and /appsody repos

.gitignore

@@ -0,0 +1,3 @@
+/metrics/appsody_reports
+/metrics/node_modules
+.DS_Store

README.md

@@ -1,2 +1,33 @@
 # utils
+
+## metrics
 Repository to store various utilities related to Appsody project
+
+
+### Data Pulling scripts
+
+In order to execute any of these scripts, you'll need to first export your GitHub API Key to an environment variable called `GITHUB_API_KEY`.
+
+  - run_all.sh
+
+    Usage:
+      - Can be passed a date in the form of `YYY-MM-DD`. This date would have to correspond to a folder in `metrics/appsody_reports/` which contains data from a previous run, to provide the total amount downloads (dockerhub & GitHub), or gained/lost (stars/watchers/forks) since that date. 
+
+      - For example: `sh run_all.sh 2020-02-10` would give the results of the current most up-to-date information, as well as a data compared to the results on the 10th of February 2020.
+
+    Runs all of the `.js` scripts in this folder consecutively, then runs `../data_pulling_utils/json_report_to_csv.py` which creates a `.csv` file for any comparisons again'st the given date (if the `.json` files from that date are found).
+
+  - appsody_dockerhub.js
+
+    Gets the pull count for each of the repositories on [appsody's dockerhub](https://hub.docker.com/u/appsody), and when the repository was last updated.
+
+  - appsody_releases.js
+
+    Gets the the download count for each of the binaries for every release of appsody from [the GitHub releases](https://github.com/appsody/appsody/releases).
+
+  - appsody_stars_watchers_forks.js
+
+    Gets the number of stars, watchers, and forks for the [appsody/stacks repository](https://github.com/appsody/stacks) and the [appsody/appsody repository](https://github.com/appsody/appsody).
+
+
+All of the results are stored in an `appsody_reports/DATE/` folder.

metrics/data_pulling_scripts/appsody_dockerhub.js

@@ -0,0 +1,43 @@
+const axios = require('axios');
+const tools = require('../data_pulling_utils/data_pulling_tools');
+
+const args = process.argv.slice(2)
+
+retrieveData();
+
+ async function retrieveData() {
+
+
+    const AuthStr = `token ${process.env.GITHUB_API_KEY}`
+    const URL = 'https://hub.docker.com/v2/repositories/appsody/?page=1&page_size=100';
+
+    axios.get(URL, { headers: { Authorization: AuthStr } })
+    .then(response => {
+
+        dockerHubResultsArray = [];
+        for (let item of response.data.results) {
+
+            const date = new Date(item.last_updated)
+            var readableTimestamp = tools.addZeroPrefix(date.getDate()) + "-" + tools.addZeroPrefix((date.getMonth() +1)) + "-" + tools.addZeroPrefix(date.getFullYear());
+
+            const single = {
+                "name": item.name,
+                "pull_count": item.pull_count,
+                "last_updated": readableTimestamp
+            }
+            dockerHubResultsArray.push(single);
+        }
+
+        tools.createLogFile("dockerhub_appsody.json", dockerHubResultsArray, function(err) {
+            console.log(err);
+         });
+
+         if(args[0] != undefined) {
+             tools.createComparisonFile("dockerhub_appsody", dockerHubResultsArray, "name", args[0]);
+         }
+
+    })
+    .catch((error) => {
+        console.log('error ' + error);
+    });
+ }

metrics/data_pulling_scripts/appsody_releases.js

@@ -0,0 +1,44 @@
+const axios = require('axios');
+const tools = require('../data_pulling_utils/data_pulling_tools');
+
+const args = process.argv.slice(2)
+
+retrieveData();
+
+ async function retrieveData() {
+
+
+    const AuthStr = `token ${process.env.GITHUB_API_KEY}`
+    const URL = 'https://api.github.com/repos/appsody/appsody/releases';
+
+    axios.get(URL, { headers: { Authorization: AuthStr } })
+    .then(response => {
+        releaseResultsArray = [];
+        for (let item of response.data) {
+
+            let name = item.tag_name
+
+            for (let asset of item.assets) {
+                const single = {
+                "release": name,
+                "cli_binary": asset.name,
+                "download_count": asset.download_count
+                }
+
+                releaseResultsArray.push(single);
+            }
+        }
+
+        tools.createLogFile("releases_appsody.json", releaseResultsArray, function(err) {
+            console.log(err);
+         });
+
+         if(args[0] != undefined) {
+             tools.createComparisonFile("releases_appsody", releaseResultsArray, "cli_binary", args[0]);
+         }
+
+    })
+    .catch((error) => {
+        console.log('error ' + error);
+    });
+ }

metrics/data_pulling_scripts/appsody_stars_watchers_forks.js

@@ -0,0 +1,79 @@
+const { graphql } = require('@octokit/graphql')
+const tools = require('../data_pulling_utils/data_pulling_tools');
+
+const args = process.argv.slice(2)
+
+async function asyncForEach(array, callback) {
+  for (let index = 0; index < array.length; index++) {
+      await callback(array[index], index, array);
+  }
+}
+
+const queryTotals = async (repo, org) => {
+  const graphqlWithAuth = graphql.defaults({
+      headers: {
+          authorization: `token ${process.env.GITHUB_API_KEY}`
+      }
+  })
+  let data
+  try {
+      data = await graphqlWithAuth(`
+  {
+    repository(name: "${repo}", owner: "${org}") {
+      forks {
+        totalCount
+      }
+      stargazers {
+        totalCount
+      }
+      watchers {
+        totalCount
+      }
+    }
+  }`)
+  } catch (err) {
+      if (err.name === 'HttpError') {
+          setTimeout(() => {
+              queryTotals(repo, org)
+          }, 10000)
+      } else {
+          throw (err)
+      }
+  } finally {
+
+      const single = {
+          "repo": repo,
+          "stars": data.repository.stargazers.totalCount,
+          "watchers": data.repository.watchers.totalCount,
+          "forks": data.repository.forks.totalCount,
+      }
+      return single;
+  }
+}
+
+var queries = [
+  ['appsody', 'stacks'],
+  ['appsody', 'appsody']
+];
+results = [];
+
+const callQueries = async () => {
+
+  const start = async () => {
+      await asyncForEach(queries, async (q) => {
+          let res = await (queryTotals(q[1], q[0]));
+          results.push(res)
+      });
+
+      tools.createLogFile(`stars_watchers_forks.json`, results, function(err) {
+          console.log(err);
+      });
+
+      if(args[0] != undefined) {
+        tools.createComparisonFile("stars_watchers_forks", results, "repo", args[0]);
+      }
+  }
+  start();
+}
+
+callQueries();

metrics/data_pulling_scripts/run_all.sh

@@ -0,0 +1,14 @@
+COMPARISON=$1
+
+for entry in ./*.js
+do
+  echo "$entry"
+  if [ -z "$COMPARISON" ]
+  then
+    `node $entry`
+  else
+    echo "Comparison $COMPARISON was given"
+    `node $entry $COMPARISON`
+  fi
+done
+`python ../data_pulling_utils/json_report_to_csv.py ../appsody_reports/`

metrics/data_pulling_utils/data_pulling_tools.js

@@ -0,0 +1,164 @@
+const fs = require('fs');
+const moment = require('moment');
+
+const folderPath = "../appsody_reports/"
+
+module.exports = {
+    createLogFile: function(extension, array, cb) {
+
+        ensurePathExists(folderPath, 0744, function(err) {
+            if (err) {
+                cb(err)
+            }
+        });
+
+        d = new Date().toISOString().slice(0, 10);
+
+        var filePath = `${folderPath}${d}/`
+        ensurePathExists(filePath, 0744, function(err) {
+            if (err) {
+                cb(err)
+            } else {
+                var filename = `${filePath}${extension}`
+
+                writeFile(filename, array, function(err) {
+                    if (err) {
+                        cb(err)
+                    }
+                });
+            }
+        })
+    },
+    addZeroPrefix: function(num) {
+        if (num.toString().length < 2) {
+            return `0${num}`;
+        } else {
+            return num;
+        }
+    },
+    createComparisonFile: function(filename, newRes, id, previous) {
+        oldResPath = `../appsody_reports/${previous}/${filename}.json`;
+        fs.access(oldResPath, fs.F_OK, (err) => {
+            if (err) {
+                console.error("No file found to make comparison.")
+                return
+            }
+
+            let rawdata = fs.readFileSync(oldResPath);
+            let oldRes = JSON.parse(rawdata); 
+
+            var comparison = compareToLast(oldRes, newRes, id);
+            this.createLogFile(`${filename}_comparison.json`, comparison, function(err) {
+                console.log(err);
+            });
+        })
+    }
+};
+
+function ensurePathExists(path, mask, cb) {
+    if (typeof mask == 'function') {
+        cb = mask;
+        mask = 0777;
+    }
+    fs.mkdir(path, mask, function(err) {
+        if (err) {
+            if (err.code == 'EEXIST') {
+                cb(null);
+            } else {
+                cb(err);
+            }
+        } else {
+            cb(null);
+        }
+    });
+}
+
+function writeFile(filename, array, cb) {
+    fs.writeFile(filename, JSON.stringify(array, null, 4), 'utf8', (err) => {
+        if (err) {
+            console.log("An error occured while writing JSON Object to File.");
+            cb(err);
+        } else {
+            cb(null);
+        }
+    });
+}
+
+function compareToLast(oldRes, newRes, id) {
+    comparisons = [];
+
+    for (let i = 0; i < newRes.length; i++) {
+        let single = {};
+        switch (id) {
+            case "repo":
+
+                var noMatch = true;
+                for (let j = 0; j < oldRes.length; j++) {
+                    if (newRes[i].repo === oldRes[j].repo) {
+                        noMatch = false;
+
+                        single = {
+                            "repo": newRes[i].repo,
+                            "stars": newRes[i].stars - oldRes[j].stars,
+                            "watchers": newRes[i].watchers - oldRes[j].watchers,
+                            "forks": newRes[i].forks - oldRes[j].forks
+                        }
+                    }
+                }
+                if (noMatch) {
+                    single = newRes[i];
+                }
+                comparisons.push(single);
+                break;
+            case "name":
+                var noMatch = true;
+                for (let j = 0; j < oldRes.length; j++) {
+                    if (newRes[i].name === oldRes[j].name) {
+                        noMatch = false;
+
+                        single = {
+                            "name": newRes[i].name,
+                            "pull_count": newRes[i].pull_count - oldRes[j].pull_count,
+                            "last_updated": newRes[i].last_updated
+                        }
+                    }
+                }
+                if (noMatch) {
+
+                    single = {
+                        "name": newRes[i].name,
+                        "pull_count": newRes[i].pull_count,
+                        "last_updated": newRes[i].last_updated
+                    }
+                }
+
+                comparisons.push(single);
+                break;
+
+            case "cli_binary":
+            var noMatch = true;
+            for (let j = 0; j < oldRes.length; j++) {
+                if (newRes[i].cli_binary === oldRes[j].cli_binary && newRes[i].release === oldRes[j].release) {
+                    noMatch = false;
+
+                    single = {
+                        "release": newRes[i].release,
+                        "cli_binary": newRes[i].cli_binary,
+                        "download_count": newRes[i].download_count - oldRes[j].download_count
+                    }
+                }
+            }
+            if (noMatch) {
+                single = {
+                    "release": newRes[i].release,
+                    "cli_binary": newRes[i].cli_binary,
+                    "download_count": newRes[i].download_count
+                }
+            }
+
+            comparisons.push(single);
+            break;
+        }
+    }
+    return comparisons;
+}