feat: more automated awesome-ipfs PRs (#429)

* chore: ignore files created by awesome_bot

* feat: auto-update README.md when PRs are created

* feat: use danger to post awesome_bot results on PRs

* chore: wut

* Update pull_requests.yml

* chore: fake-test

* adjust branch used by github-push-action

* chore: ensure checkout uses the branch

* chore: update README.md

* chore: modify dangerfile and workflow ordering

* remove fake addition to videos.yaml

* chore: update README.md

* chore: rename pull_requests.yml

* temp: always display awesomeBot results

* Revert "temp: always display awesomeBot results"

This reverts commit acaef1f4f2.

* chore(deps): install danger as devDep

* feat: cache node_modules and readme

* Update pull_requests.yml

* chore: remove ad-m/github-push-action

see https://github.com/ad-m/github-push-action/issues/104

* tmp: PR-demo

* Rebuild

* Revert "tmp: PR-demo"

This reverts commit 8bed1326f0.

* Rebuild

* fix: no need to push readme update

* fix: no need to push readme update

readme is updated during 'Rebuild' commit from data.yml

* chore: run danger via npm script instead of npx

* feat: remove replace circleci cron with github action

this github action should open a PR removing broken links automatically

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: GitHub Action <action@github.com>

.circleci/config.yml

@@ -1,36 +0,0 @@
-version: 2
-jobs:
-  awesome:
-    docker:
-      - image: circleci/ruby:2.4.2-jessie-node
-    steps:
-      - checkout
-      - run: gem install awesome_bot
-      - run: awesome_bot --allow 429 --allow-redirect --allow-dupe --allow-ssl -w ipfs.io README.md
-
-  build:
-    docker:
-      - image: circleci/node:10.15.1-browsers
-        environment:
-          NO_SANDBOX: true
-    steps:
-      - checkout
-      - run: npm ci
-      - run: npm run build
-
-workflows:
-  version: 2
-  check-links:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - master
-    jobs:
-      - awesome
-  checks:
-    jobs:
-      - awesome
-      - build

.github/workflows/broken_link_cron.yml

@@ -0,0 +1,82 @@
+name: Open a PR to remove broken links on a schedule
+
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: '0 0 * * *'
+
+jobs:
+  check-for-broken-links:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 13
+
+    - uses: ipfs/aegir/actions/cache-node-modules@master
+      with:
+        directories: |
+          README.md
+        build: |
+          npm run build:readme
+        cache_name: readme
+
+    - name: Set up Ruby 3.1.2
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.1.2
+
+    - name: Install and run awesome_bot
+      run: |
+        gem install awesome_bot
+        awesome_bot --allow 429 --allow-redirect --allow-dupe --allow-ssl -w ipfs.io README.md || echo
+
+    - name: Remove broken links
+      run: |
+        node scripts/remove-broken-links.js
+
+    - uses: stefanzweifel/git-auto-commit-action@49620cd3ed21ee620a48530e81dba0d139c9cb80
+      with:
+        # Optional. Commit message for the created commit.
+        # Defaults to "Apply automatic changes"
+        commit_message: "chore: Remove broken links"
+
+        # Optional. Local and remote branch name where commit is going to be pushed
+        #  to. Defaults to the current branch.
+        #  You might need to set `create_branch: true` if the branch does not exist.
+        branch: remove-broken-links-${{ github.run_id }}
+
+        commit_options: '--no-verify --signoff'
+
+        # Optional glob pattern of files which should be added to the commit
+        # Defaults to all (.)
+        # See the `pathspec`-documentation for git
+        # - https://git-scm.com/docs/git-add#Documentation/git-add.txt-ltpathspecgt82308203
+        # - https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefpathspecapathspec
+        file_pattern: data/*
+        repository: .
+        add_options: '-A'
+        push_options: '--force'
+        skip_dirty_check: true
+        skip_fetch: true
+        skip_checkout: true
+        disable_globbing: true
+        create_branch: true
+
+    - name: pull-request
+      uses: repo-sync/pull-request@65785d95a5a466e46a9d0708933a3bd51bbf9dde
+      with:
+        source_branch: "remove-broken-links-${{ github.run_id }}"
+        destination_branch: "master"
+        pr_title: "chore: Remove broken links"
+        pr_body: "Automated PR created by .github/workflows/broken_link_cron.yml"
+        pr_label: "kind/maintenance"
+        pr_draft: false
+        pr_allow_empty: false
+        github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pull_requests.yml

@@ -0,0 +1,60 @@
+
+name: Pull request workflow
+
+on:
+  pull_request:
+    types: [ opened, synchronize, reopened ]
+    branches: [ master, main ]
+
+jobs:
+  # Run linting and tests
+  build-and-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 13
+      - uses: ipfs/aegir/actions/cache-node-modules@master
+        with:
+          directories: |
+            README.md
+          build: |
+            npm run build:readme
+          cache_name: readme
+      # aegir/actions/cache-node-modules runs build. We don't need to run it again.
+      - run: npm run lint
+
+  run-danger:
+    runs-on: ubuntu-latest
+    needs: build-and-lint
+    if: "!contains(github.event.head_commit.message, '[skip ci]')"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 13
+      - name: Set up Ruby 3.1.2
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.1.2
+      - uses: ipfs/aegir/actions/cache-node-modules@master
+        with:
+          directories: |
+            README.md
+          build: |
+            npm run build:readme
+          cache_name: readme
+      - name: Install and run awesome_bot
+        # It's okay for awesome_bot to fail. Danger will post a comment on the PR if it fails.
+        run: |
+          gem install awesome_bot
+          awesome_bot --allow 429 --allow-redirect --allow-dupe --allow-ssl -w ipfs.io README.md || echo
+      - name: Run danger
+        run: npm run ci:danger
+        env:
+          DANGER_GITHUB_API_TOKEN: ${{ secrets.DANGER_GITHUB_API_TOKEN }}

.gitignore

@@ -30,3 +30,9 @@ dist
 /src/static/fonts
 /src/static/app.css
 /src/static/app.js
+
+# awesome_bot output
+ab-results-README.md-filtered.json
+ab-results-README.md.json
+ab-results-README.md-markdown-table.json
+links.txt

dangerfile.js

@@ -0,0 +1,30 @@
+const { danger, markdown, fail, warn, message } = require('danger')
+const awesomeBotResults = require('./ab-results-README.md-markdown-table.json')
+
+const githubMetadata = danger.git.fileMatch('.github/**')
+const yamlData = danger.git.fileMatch('data/**')
+const scripts = danger.git.fileMatch('scripts/**')
+const src = danger.git.fileMatch('src/**')
+
+if (githubMetadata.edited) {
+  message('Changes were made within the .github folder.')
+}
+
+if (yamlData.edited) {
+  message('YAML data files were edited.')
+  /**
+   * Only display awesomeBot results if the data yaml used to generate the readme have been modified.
+   */
+  if (awesomeBotResults.error) {
+    fail(awesomeBotResults.title)
+    markdown(awesomeBotResults.message)
+  }
+}
+
+if (scripts.edited) {
+  warn('Changes were made to the scripts folder.')
+}
+
+if (src.edited) {
+  warn('Changes were made to the src folder.')
+}

package.json

@@ -18,13 +18,15 @@
     "build:css": "postcss --no-map --use postcss-import cssnano -o src/static/app.css src/css/*.css",
     "build:js": "browserify -g uglifyify src/js/app.js -o src/static/app.js",
     "build:data": "node ./scripts/make-data.js",
-    "build:hugo": "hugo -s src -d ../public --cleanDestinationDir --minify --gc"
+    "build:hugo": "hugo -s src -d ../public --cleanDestinationDir --minify --gc",
+    "ci:danger": "danger ci"
   },
   "devDependencies": {
     "babel-eslint": "^10.0.2",
     "browserify": "^16.3.0",
     "chokidar": "^3.0.2",
     "cssnano": "^4.1.10",
+    "danger": "^11.1.2",
     "ecstatic": "^4.1.2",
     "fs-extra": "^8.1.0",
     "hugo-bin": "^0.43.6",

scripts/remove-broken-links.js

@@ -0,0 +1,28 @@
+/**
+ * This file assumes that awesome_bot has already been ran.
+ */
+const markdownTable = require('../ab-results-README.md-markdown-table.json')
+const fs = require('fs').promises
+const { join } = require('path')
+
+const dir = join(__dirname, '../data')
+
+if (markdownTable.error) {
+  const brokenLinks = require('../ab-results-README.md-filtered.json').map((resultsData) => resultsData.link);
+  (async () => {
+    const files = (await fs.readdir(dir)).map(file => join(dir, file))
+
+    for await (const filePath of files) {
+      let fileContents = await fs.readFile(filePath, 'utf8')
+      brokenLinks.forEach((brokenLink) => {
+        console.log('Searching for "' + brokenLink + '" in ' + filePath)
+        const regex = new RegExp(brokenLink, 'g')
+        if (fileContents.match(regex)) {
+          fileContents = fileContents.replace(regex, '')
+          console.log('Removed "' + brokenLink + '" from ' + filePath)
+        }
+      })
+      await fs.writeFile(filePath, fileContents, 'utf8')
+    }
+  })()
+}