diff --git a/setup/ansible-inventory b/setup/ansible-inventory index b08778bdf..e34a13656 100644 --- a/setup/ansible-inventory +++ b/setup/ansible-inventory @@ -111,7 +111,7 @@ test-nodesource_piccoloaiutante-debian8-arm_pi3-1 iojs-www [node-www] -node-www +infra-digitalocean-ubuntu1604-x64-1 [iojs-jenkins] iojs-jenkins diff --git a/setup/www/ansible-playbook.retry b/setup/www/ansible-playbook.retry new file mode 100644 index 000000000..aba078e00 --- /dev/null +++ b/setup/www/ansible-playbook.retry @@ -0,0 +1 @@ +infra-digitalocean-ubuntu1604-x64-1 diff --git a/setup/www/ansible-playbook.yaml b/setup/www/ansible-playbook.yaml index d0aa1a3a6..f97b0955d 100644 --- a/setup/www/ansible-playbook.yaml +++ b/setup/www/ansible-playbook.yaml @@ -24,14 +24,11 @@ - include: tasks/nginx.yaml tags: nginx - - include: tasks/dist-indexer.yaml - tags: dist-indexer - - - include: tasks/nightly-builder.yaml - tags: nightly-builder - - include: tasks/promote.yaml tags: promote - - include: tasks/latest-linker.yaml - tags: latest-linker + - include: tasks/tools.yaml + tags: tools + + - include: tasks/metrics.yaml + tags: metrics diff --git a/setup/www/ansible-vars.yaml b/setup/www/ansible-vars.yaml index f9c835d7f..00a8872ff 100644 --- a/setup/www/ansible-vars.yaml +++ b/setup/www/ansible-vars.yaml @@ -1,16 +1,14 @@ --- server_user: nodejs -root_users: - - rvagg - - jbergstroem - - orangemocha dist_users: - - cjihrig - chrisdickinson - - sam-github - - fishrock123 + - cjihrig + - evanlucas - jasnell + - fishrock123 + - thealphanerd - rvagg + - sam-github libuv_users: - saghul - piscisaureus @@ -19,3 +17,6 @@ packages: - nodejs - nginx - git + - python-pip + - gnuplot +benchmark_host: 50.97.245.4 \ No newline at end of file diff --git a/setup/www/host_vars/.gitignore b/setup/www/host_vars/.gitignore index 549628fb0..22cdad5a0 100644 --- a/setup/www/host_vars/.gitignore +++ b/setup/www/host_vars/.gitignore @@ -1 +1 @@ -node-www \ No newline at end of file +infra-* \ No newline at end of file diff --git a/setup/www/host_vars/node-www.tmpl b/setup/www/host_vars/node-www.tmpl index b905f4d5e..ce7577199 100644 --- a/setup/www/host_vars/node-www.tmpl +++ b/setup/www/host_vars/node-www.tmpl @@ -4,5 +4,4 @@ cdn_api_key: "INSERT CLOUDFLARE API KEY" cdn_api_email: "INSERT CLOUDFLARE API EMAIL" cdn_api_iojs_id: "INSERT CLOUDFLARE API iojs.org ID" cdn_api_nodejs_id: "INSERT CLOUDFLARE API nodejs.org ID" -cloudfuse_user: "INSERT RACKSPACE CLOUD USERNAME" -cloudfuse_key: "INSERT RACKSPACE CLOUD API KEY" +jenkins_token: "INSERT JENKINS API TOKEN" diff --git a/setup/www/resources/.gitignore b/setup/www/resources/.gitignore index 059c69adf..fa375ddc2 100644 --- a/setup/www/resources/.gitignore +++ b/setup/www/resources/.gitignore @@ -1,4 +1,3 @@ *.crt *.key *.pem -secrets/* diff --git a/setup/www/resources/config/.cloudfuse b/setup/www/resources/config/.cloudfuse deleted file mode 100644 index 7b521149a..000000000 --- a/setup/www/resources/config/.cloudfuse +++ /dev/null @@ -1,3 +0,0 @@ -username={{ cloudfuse_user }} -api_key={{ cloudfuse_key }} -region=IAD \ No newline at end of file diff --git a/setup/www/resources/config/benchmarking.nodejs.org b/setup/www/resources/config/benchmarking.nodejs.org index 2f99cb9d5..13129e140 100644 --- a/setup/www/resources/config/benchmarking.nodejs.org +++ b/setup/www/resources/config/benchmarking.nodejs.org @@ -1,32 +1,19 @@ server { listen *:80; + listen [::]:80; server_name benchmarking.nodejs.org; return 301 https://benchmarking.nodejs.org$request_uri; } server { - listen [::]:443 ssl spdy; - listen *:443 ssl spdy; + listen [::]:443 ssl http2; + listen *:443 ssl http2; server_name benchmarking.nodejs.org; ssl_certificate ssl/nodejs_chained.crt; ssl_certificate_key ssl/nodejs.key; ssl_trusted_certificate ssl/nodejs_chained.crt; - ssl_dhparam ssl/dhparam.pem; - - ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS; - ssl_prefer_server_ciphers on; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - - ssl_session_cache shared:benchmarking:100m; - ssl_session_timeout 24h; - - ssl_stapling on; - ssl_stapling_verify on; - - spdy_keepalive_timeout 300; - spdy_headers_comp 9; keepalive_timeout 60; server_tokens off; @@ -38,13 +25,13 @@ server { add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; - access_log /var/log/nginx/benchmarking.nodejs.org-access.log nodejs; - error_log /var/log/nginx/benchmarking.nodejs.org-error.log; + access_log /var/log/nginx/benchmarking/benchmarking.nodejs.org-access.log nodejs; + error_log /var/log/nginx/benchmarking/benchmarking.nodejs.org-error.log; gzip on; gzip_static on; gzip_disable "MSIE [1-6]\."; - gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript; + gzip_types text/plain text/css application/javascript text/xml application/xml application/xml+rss image/svg+xml; root /home/www/benchmarking; default_type text/plain; diff --git a/setup/www/resources/config/coverage.nodejs.org b/setup/www/resources/config/coverage.nodejs.org new file mode 100644 index 000000000..f567fe6aa --- /dev/null +++ b/setup/www/resources/config/coverage.nodejs.org @@ -0,0 +1,39 @@ +server { + listen *:80; + listen [::]:80; + server_name coverage.nodejs.org; + + return 301 https://coverage.nodejs.org$request_uri; +} + +server { + listen [::]:443 ssl http2; + listen *:443 ssl http2; + server_name coverage.nodejs.org; + + ssl_certificate ssl/nodejs_chained.crt; + ssl_certificate_key ssl/nodejs.key; + ssl_trusted_certificate ssl/nodejs_chained.crt; + + keepalive_timeout 60; + server_tokens off; + + resolver 8.8.4.4 8.8.8.8 valid=300s; + resolver_timeout 10s; + + add_header Strict-Transport-Security max-age=63072000; + add_header X-Frame-Options DENY; + add_header X-Content-Type-Options nosniff; + + access_log /var/log/nginx/coverage/coverage.nodejs.org-access.log nodejs; + error_log /var/log/nginx/coverage/coverage.nodejs.org-error.log; + + gzip on; + gzip_static on; + gzip_disable "MSIE [1-6]\."; + gzip_types text/plain text/css application/javascript text/xml application/xml application/xml+rss image/svg+xml; + + root /home/www/coverage; + default_type text/plain; + index index.html; +} diff --git a/setup/www/resources/config/github-webhook.conf b/setup/www/resources/config/github-webhook.conf deleted file mode 100644 index d09aaafe4..000000000 --- a/setup/www/resources/config/github-webhook.conf +++ /dev/null @@ -1,9 +0,0 @@ -start on runlevel [2345] -stop on runlevel [!2345] - -respawn -respawn limit 10 5 - -setuid nodejs - -exec github-webhook --config /etc/github-webhook.json diff --git a/setup/www/resources/config/github-webhook.json b/setup/www/resources/config/github-webhook.json.j2 similarity index 94% rename from setup/www/resources/config/github-webhook.json rename to setup/www/resources/config/github-webhook.json.j2 index dec65cf8d..23c283a3b 100644 --- a/setup/www/resources/config/github-webhook.json +++ b/setup/www/resources/config/github-webhook.json.j2 @@ -1,7 +1,7 @@ { "port": 9999, "path": "/webhook", - "secret": "{{github_secret}}", + "secret": "{{ github_secret }}", "log": "/home/nodejs/github-webhook.log", "rules": [ { diff --git a/setup/www/resources/config/github-webhook.service.j2 b/setup/www/resources/config/github-webhook.service.j2 new file mode 100644 index 000000000..85b7c8ea2 --- /dev/null +++ b/setup/www/resources/config/github-webhook.service.j2 @@ -0,0 +1,21 @@ +[Unit] +Description=GitHub Webhook +Wants=network.target +After=network.target + +[Install] +WantedBy=multi-user.target + +[Service] +Type=simple +User={{ server_user }} + +Environment="USER={{ server_user }}" +Environment="SHELL=/bin/bash" +Environment="HOME=/home/{{ server_user }}" +Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +ExecStart=/usr/bin/github-webhook --config /etc/github-webhook.json +Restart=always +RestartSec=30 +StartLimitInterval=0 diff --git a/setup/www/resources/config/iojs.org b/setup/www/resources/config/iojs.org index ea3be3348..c0b633d88 100644 --- a/setup/www/resources/config/iojs.org +++ b/setup/www/resources/config/iojs.org @@ -6,8 +6,8 @@ server { } server { - listen [::]:443 ssl spdy; - listen *:443 ssl spdy; + listen [::]:443 ssl http2; + listen *:443 ssl http2; server_name www.iojs.org; ssl_certificate ssl/iojs_chained.crt; @@ -18,8 +18,8 @@ server { } server { - listen [::]:443 ssl spdy; - listen *:443 ssl spdy; + listen [::]:443 ssl http2; + listen *:443 ssl http2; server_name iojs.org; ssl_certificate ssl/iojs_chained.crt; @@ -36,13 +36,13 @@ server { add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; - access_log /var/log/nginx/iojs.org-access.log nodejs; - error_log /var/log/nginx/iojs.org-error.log; + access_log /var/log/nginx/iojs/iojs.org-access.log nodejs; + error_log /var/log/nginx/iojs/iojs.org-error.log; gzip on; gzip_static on; gzip_disable "MSIE [1-6]\."; - gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript; + gzip_types text/plain text/css application/javascript text/xml application/xml application/xml+rss image/svg+xml; root /home/www/iojs; default_type text/plain; diff --git a/setup/www/resources/config/libuv.org b/setup/www/resources/config/libuv.org index 243faeb64..272e5576d 100644 --- a/setup/www/resources/config/libuv.org +++ b/setup/www/resources/config/libuv.org @@ -1,5 +1,6 @@ server { listen *:80; + listen [::]:80; server_name dist.libuv.org; keepalive_timeout 60; @@ -11,13 +12,13 @@ server { add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; - access_log /var/log/nginx/libuv.org-access.log nodejs; - error_log /var/log/nginx/libuv.org-error.log; + access_log /var/log/nginx/libuv/libuv.org-access.log nodejs; + error_log /var/log/nginx/libuv/libuv.org-error.log; gzip on; gzip_static on; gzip_disable "MSIE [1-6]\."; - gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript; + gzip_types text/plain text/css application/javascript text/xml application/xml application/xml+rss image/svg+xml; root /home/libuv/www; default_type text/plain; diff --git a/setup/www/resources/config/logrotate-nginx b/setup/www/resources/config/logrotate-nginx index 03582b84a..df218e938 100644 --- a/setup/www/resources/config/logrotate-nginx +++ b/setup/www/resources/config/logrotate-nginx @@ -1,4 +1,4 @@ -/var/log/nginx/*.log { +/var/log/nginx/*.log, /var/log/nginx/nodejs/*.log, /var/log/nginx/iojs/*.log, /var/log/nginx/libuv/*.log, /var/log/nginx/benchmarking/*.log, /var/log/nginx/coverage/*.log { daily missingok compresscmd /usr/bin/xz @@ -13,6 +13,7 @@ dateyesterday maxsize 500M sharedscripts + rotate 36500 prerotate if [ -d /etc/logrotate.d/httpd-prerotate ]; then \ run-parts /etc/logrotate.d/httpd-prerotate; \ diff --git a/setup/www/resources/config/nightly-builder.json.j2 b/setup/www/resources/config/nightly-builder.json.j2 new file mode 100644 index 000000000..8a3b4af52 --- /dev/null +++ b/setup/www/resources/config/nightly-builder.json.j2 @@ -0,0 +1,11 @@ +{ + "jenkinsToken": "{{ jenkins_token }}", + "jenkinsJobUrl": "https://ci-release.nodejs.org/job/iojs+release", + "jenkinsCrumbUrl": "https://ci-release.nodejs.org/crumbIssuer/api/json", + "githubAuthUser": "{{ github_auth_user }}", + "githubAuthToken": "{{ github_auth_token }}", + "githubOrg": "nodejs", + "githubRepo": "node", + "githubScheme": "https://github.com/", + "releaseUrlBase": "https://nodejs.org/download/" +} \ No newline at end of file diff --git a/setup/www/resources/config/nodejs.org b/setup/www/resources/config/nodejs.org index 258a6f847..daee9beed 100644 --- a/setup/www/resources/config/nodejs.org +++ b/setup/www/resources/config/nodejs.org @@ -16,8 +16,8 @@ server { listen [::]:80 default_server ipv6only=on; server_name nodejs.org; - access_log /var/log/nginx/nodejs.org-access.log nodejs; - error_log /var/log/nginx/nodejs.org-error.log; + access_log /var/log/nginx/nodejs/nodejs.org-access.log nodejs; + error_log /var/log/nginx/nodejs/nodejs.org-error.log; keepalive_timeout 60; server_tokens off; @@ -25,7 +25,7 @@ server { gzip on; gzip_static on; gzip_disable "MSIE [1-6]\."; - gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript; + gzip_types text/plain text/css application/javascript text/xml application/xml application/xml+rss image/svg+xml; # let the blog.nodejs.org redirector handle this location ~ ^/blog(.*) { @@ -168,24 +168,24 @@ server { } server { - listen *:443 ssl spdy; - listen [::]:443 ssl spdy; + listen *:443 ssl http2; + listen [::]:443 ssl http2; server_name www.nodejs.org; return 301 https://nodejs.org$request_uri; } server { - listen *:443 ssl spdy; - listen [::]:443 ssl spdy; + listen *:443 ssl http2; + listen [::]:443 ssl http2; server_name blog.nodejs.org; return 301 http://blog.nodejs.org$request_uri; } server { - listen *:443 default_server ssl spdy; - listen [::]:443 default_server ipv6only=on ssl spdy; + listen *:443 default_server ssl http2; + listen [::]:443 default_server ipv6only=on ssl http2; server_name nodejs.org; @@ -199,13 +199,13 @@ server { #add_header X-Frame-Options DENY; #add_header X-Content-Type-Options nosniff; - access_log /var/log/nginx/nodejs.org-access.log nodejs; - error_log /var/log/nginx/nodejs.org-error.log; + access_log /var/log/nginx/nodejs/nodejs.org-access.log nodejs; + error_log /var/log/nginx/nodejs/nodejs.org-error.log; gzip on; gzip_static on; gzip_disable "MSIE [1-6]\."; - gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript; + gzip_types text/plain text/css application/javascript text/xml application/xml application/xml+rss image/svg+xml; error_page 404 @localized_404; diff --git a/setup/www/resources/config/nodejs_ssh_config.j2 b/setup/www/resources/config/nodejs_ssh_config.j2 new file mode 100644 index 000000000..fba643876 --- /dev/null +++ b/setup/www/resources/config/nodejs_ssh_config.j2 @@ -0,0 +1,4 @@ +Host benchmark + HostName {{ benchmark_host }} + User benchmark + IdentityFile ~/.ssh/benchmark_id_rsa \ No newline at end of file diff --git a/setup/www/resources/config/ssl-defaults.conf b/setup/www/resources/config/ssl-defaults.conf index 77344c115..4bbd2e7d2 100644 --- a/setup/www/resources/config/ssl-defaults.conf +++ b/setup/www/resources/config/ssl-defaults.conf @@ -4,7 +4,6 @@ ssl_trusted_certificate ssl/nodejs_chained.crt; ssl_dhparam ssl/dhparam.pem; ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS; -ssl_prefer_server_ciphers on; ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_session_cache shared:nodejs:376m; @@ -12,6 +11,3 @@ ssl_session_timeout 24h; ssl_stapling on; ssl_stapling_verify on; - -spdy_keepalive_timeout 300; -spdy_headers_comp 9; diff --git a/setup/www/resources/scripts/build-site.sh b/setup/www/resources/scripts/build-site.sh index d044ec16d..215524f82 100755 --- a/setup/www/resources/scripts/build-site.sh +++ b/setup/www/resources/scripts/build-site.sh @@ -34,6 +34,9 @@ git clean -fdx git fetch origin git checkout origin/master +nodeuid=$(grep ^nodejs: /etc/passwd | awk -F: '{print $3}') +nodegid=$(grep ^nodejs: /etc/passwd | awk -F: '{print $4}') + docker pull node:latest docker run \ --rm \ @@ -41,8 +44,8 @@ docker run \ -v /home/nodejs/.npm:/npm/ \ node:latest \ bash -c " \ - addgroup nodejs --gid 1000 && \ - adduser nodejs --uid 1000 --gid 1000 --gecos nodejs --disabled-password && \ + addgroup nodejs --gid ${nodeuid} && \ + adduser nodejs --uid ${nodeuid} --gid ${nodegid} --gecos nodejs --disabled-password && \ su nodejs -c ' \ npm config set loglevel http && \ npm config set cache /npm/ && \ diff --git a/setup/www/resources/scripts/cdn-purge.sh b/setup/www/resources/scripts/cdn-purge.sh.j2 similarity index 86% rename from setup/www/resources/scripts/cdn-purge.sh rename to setup/www/resources/scripts/cdn-purge.sh.j2 index 804bf8512..4717b8b3a 100644 --- a/setup/www/resources/scripts/cdn-purge.sh +++ b/setup/www/resources/scripts/cdn-purge.sh.j2 @@ -2,10 +2,10 @@ set -e -api_key="{{cdn_api_key}}" -api_email="{{cdn_api_email}}" -iojs_id="{{cdn_api_iojs_id}}" -nodejs_id="{{cdn_api_nodejs_id}}" +api_key="{{ cdn_api_key }}" +api_email="{{ cdn_api_email }}" +iojs_id="{{ cdn_api_iojs_id }}" +nodejs_id="{{ cdn_api_nodejs_id }}" site=$1 diff --git a/setup/www/resources/secrets/makesecrets.sh b/setup/www/resources/secrets/makesecrets.sh index 634648c54..106b1e06b 100755 --- a/setup/www/resources/secrets/makesecrets.sh +++ b/setup/www/resources/secrets/makesecrets.sh @@ -55,6 +55,8 @@ echo "" >> ./nodejs_chained.crt echo "Extracting staging_id_rsa_public.key..." bash -c "$gpgcmd" < ${secretsdir}/build/release/staging_id_rsa_public.key 1> ./staging_id_rsa_public.key 2> /dev/null || bork -echo "" >> ./staging_id_rsa_public.key + +echo "Extracting nodejs_build_benchmark key..." +bash -c "$gpgcmd" < ${secretsdir}/build/infra/nodejs_build_benchmark 1> ./benchmark_id_rsa.key 2> /dev/null || bork diff --git a/setup/www/tasks/base.yaml b/setup/www/tasks/base.yaml index 9fe989400..d82856513 100644 --- a/setup/www/tasks/base.yaml +++ b/setup/www/tasks/base.yaml @@ -1,5 +1,5 @@ - name: Base | Add the NodeSource Node.js repo - command: "bash -c 'curl -sL https://deb.nodesource.com/setup_4.x | bash -'" + command: "bash -c 'curl -sL https://deb.nodesource.com/setup_6.x | bash -'" tags: base - name: Base | APT Update diff --git a/setup/www/tasks/dist-indexer.yaml b/setup/www/tasks/dist-indexer.yaml deleted file mode 100644 index 7971e5d62..000000000 --- a/setup/www/tasks/dist-indexer.yaml +++ /dev/null @@ -1,30 +0,0 @@ -- name: dist-indexer | Make /home/dist/tools/dist-indexer - file: - path: /home/dist/tools/dist-indexer - state: directory - mode: 0755 - owner: dist - group: dist - tags: dist-indexer - -- name: dist-indexer | Copy source files - copy: - src: tools/dist-indexer/ - dest: /home/dist/tools/dist-indexer/ - owner: dist - group: dist - tags: dist-indexer - -- name: dist-indexer | Set executables - file: - path: /home/dist/tools/dist-indexer/{{ item }} - mode: 0755 - with_items: - - dist-indexer.js - - ls-types.js - -- name: dist-indexer | npm install - shell: cd /home/dist/tools/dist-indexer/ && npm install - become: yes - become_user: dist - tags: dist-indexer diff --git a/setup/www/tasks/latest-linker.yaml b/setup/www/tasks/latest-linker.yaml deleted file mode 100644 index 2b558e6e0..000000000 --- a/setup/www/tasks/latest-linker.yaml +++ /dev/null @@ -1,25 +0,0 @@ -- name: Site Setup | Make /home/dist/tools/latest-linker - file: - path: /home/dist/tools/latest-linker - state: directory - mode: 0755 - owner: dist - group: dist - tags: latest-linker - -- name: latest-linker | Copy source files - copy: - src: tools/latest-linker/ - dest: /home/dist/tools/latest-linker/ - owner: dist - group: dist - tags: latest-linker - -- name: latest-linker | npm install - shell: cd /home/dist/tools/latest-linker/ && npm install - tags: latest-linker - -- name: latest-linker | Set executables - file: - path: /home/dist/tools/latest-linker/latest-linker.js - mode: 0755 diff --git a/setup/www/tasks/metrics.yaml b/setup/www/tasks/metrics.yaml new file mode 100644 index 000000000..bdee6dd6b --- /dev/null +++ b/setup/www/tasks/metrics.yaml @@ -0,0 +1,32 @@ +- name: Metrics | Make /home/nodejs/metrics-bin + file: + path: /home/nodejs/metrics-bin + state: directory + mode: 0755 + owner: root + group: root + tags: metrics + +- name: Metrics | Copy source files + copy: + src: tools/metrics/ + dest: /home/nodejs/metrics-bin/ + owner: root + group: root + mode: 0750 + tags: metrics + +- name: Metrics | Install geoip2 Python module + pip: + name: geoip2 + tags: metrics + +- name: Metrics | Install latest GeoLite2 City database + command: "bash -c 'curl -sL http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz | gunzip -c - > /home/nodejs/metrics-bin/GeoLite2-City.mmdb'" + tags: metrics + +- name: Metrics | Add periodic task to crontab + lineinfile: + dest: /etc/crontab + line: "0 */4 * * * root /home/nodejs/metrics-bin/download-counts.sh" + tags: metrics diff --git a/setup/www/tasks/nginx.yaml b/setup/www/tasks/nginx.yaml index e92a9ead0..8f24413bd 100644 --- a/setup/www/tasks/nginx.yaml +++ b/setup/www/tasks/nginx.yaml @@ -8,6 +8,7 @@ - iojs.org - libuv.org - benchmarking.nodejs.org + - coverage.nodejs.org tags: nginx - name: nginx | Create nginx config symlinks @@ -20,6 +21,7 @@ - { src: iojs.org, dest: 01-iojs.org } - { src: libuv.org, dest: 02-libuv.org } - { src: benchmarking.nodejs.org, dest: 03-benchmarking.nodejs.org } + - { src: coverage.nodejs.org, dest: 03-coverage.nodejs.org } tags: nginx - name: nginx | Make /etc/nginx/ssl/ @@ -47,6 +49,14 @@ copy: src: ./resources/config/ssl-defaults.conf dest: /etc/nginx/conf.d/ssl-defaults.conf + tags: nginx + +- name: nginx | Remove ssl_protocols from default nginx config + replace: + dest: /etc/nginx/nginx.conf + regexp: '^(\s+)ssl_protocols' + replace: '\1# ssl_protocols' + tags: nginx - name: nginx | Install open file cache config copy: @@ -81,8 +91,24 @@ group: root tags: nginx +- name: nginx | Make /var/log/nginx/ subdirectories + file: + path: '/var/log/nginx/{{ item }}' + state: directory + mode: 0755 + owner: www-data + group: adm + with_items: + - 'nodejs' + - 'iojs' + - 'libuv' + - 'benchmarking' + - 'coverage' + tags: nginx + - name: nginx | Restart service service: name: nginx state: restarted + enabled: yes tags: nginx diff --git a/setup/www/tasks/nightly-builder.yaml b/setup/www/tasks/nightly-builder.yaml deleted file mode 100644 index c04159a73..000000000 --- a/setup/www/tasks/nightly-builder.yaml +++ /dev/null @@ -1,20 +0,0 @@ -- name: Site Setup | Make /home/dist/tools/nightly-builder - file: - path: /home/dist/tools/nightly-builder - state: directory - mode: 0755 - owner: dist - group: dist - tags: nightly-builder - -- name: nightly-builder | Copy source files - copy: - src: tools/nightly-builder/ - dest: /home/dist/tools/nightly-builder/ - owner: dist - group: dist - tags: nightly-builder - -- name: nightly-builder | npm install - shell: cd /home/dist/tools/nightly-builder/ && npm install - tags: nightly-builder diff --git a/setup/www/tasks/site-setup.yaml b/setup/www/tasks/site-setup.yaml index c2b7ff730..11875a2c3 100644 --- a/setup/www/tasks/site-setup.yaml +++ b/setup/www/tasks/site-setup.yaml @@ -11,38 +11,54 @@ - check-build-site.sh tags: setup -- name: Site Setup | Copy cdn-purge.sh script - copy: - src: ./resources/scripts/cdn-purge.sh - dest: /home/nodejs/cdn-purge.sh +- name: Site Setup | Make /home/www + file: + path: "{{ item }}" + state: directory + mode: 0755 + owner: nodejs + with_items: + - /home/www + - /home/www/iojs + - /home/www/nodejs + - /home/www/github + - /home/www/benchmarking + - /home/www/coverage + tags: setup + +- name: Site Setup | Make /home/nodejs/.ssh + file: + path: /home/nodejs/.ssh + state: directory mode: 0700 - owner: root - group: root + owner: nodejs tags: setup -- name: Site Setup | Replace cdn-purge.sh secrets - replace: - dest: /home/nodejs/cdn-purge.sh - regexp: "{{ item.regexp }}" - replace: "{{ item.replace }}" - with_items: - - { regexp: '\{\{cdn_api_key\}\}', replace: '{{ cdn_api_key }}' } - - { regexp: '\{\{cdn_api_email\}\}', replace: '{{ cdn_api_email }}' } - - { regexp: '\{\{cdn_api_iojs_id\}\}', replace: '{{ cdn_api_iojs_id }}' } - - { regexp: '\{\{cdn_api_nodejs_id\}\}', replace: '{{ cdn_api_nodejs_id }}' } +- name: Site Setup | Copy benchmark_id_rsa key + copy: + src: ./resources/secrets/benchmark_id_rsa.key + dest: /home/nodejs/.ssh/benchmark_id_rsa + mode: 0600 + owner: nodejs + group: nodejs tags: setup -- name: Site Setup | Add periodic tasks to crontab - lineinfile: - dest: /etc/crontab - line: "{{ item }}" - with_items: - - '*/5 * * * * dist /home/staging/tools/promote/promote_nightly.sh nodejs' - - '*/5 * * * * dist /home/staging/tools/promote/promote_nightly.sh iojs' - - '* * * * * nodejs /home/nodejs/check-build-site.sh nodejs' - - '* * * * * nodejs /home/nodejs/check-build-site.sh iojs' - - '* * * * * root /home/nodejs/cdn-purge.sh' - - '* */4 * * * nodejs rsync -aqz --delete benchmark:charts/ /home/www/benchmarking/charts/' +- name: Site Setup | Copy nodejs SSH config + template: + src: ./resources/config/nodejs_ssh_config.j2 + dest: /home/nodejs/.ssh/config + mode: 0600 + owner: nodejs + group: nodejs + tags: setup + +- name: Site Setup | Copy cdn-purge.sh script + template: + src: ./resources/scripts/cdn-purge.sh.j2 + dest: /home/nodejs/cdn-purge.sh + mode: 0700 + owner: root + group: root tags: setup - name: Site Setup | Copy queue-cdn-purge.sh script @@ -54,20 +70,6 @@ group: root tags: setup -- name: Site Setup | Make /home/www - file: - path: "{{ item }}" - state: directory - mode: 0755 - owner: nodejs - with_items: - - /home/www - - /home/www/iojs - - /home/www/nodejs - - /home/www/github - - /home/www/benchmarking - tags: setup - - name: Site Setup | Make /home/nodejs/.npm file: path: /home/nodejs/.npm @@ -85,3 +87,17 @@ - "/home/nodejs/build-site.sh iojs" - "/home/nodejs/build-benchmarking-site.sh" tags: setup + +- name: Site Setup | Add periodic tasks to crontab + lineinfile: + dest: /etc/crontab + line: "{{ item }}" + with_items: + - '*/5 * * * * dist /home/staging/tools/promote/promote_nightly.sh nodejs' + - '*/5 * * * * dist /home/staging/tools/promote/promote_nightly.sh iojs' + - '* * * * * nodejs /home/nodejs/check-build-site.sh nodejs' + - '* * * * * nodejs /home/nodejs/check-build-site.sh iojs' + - '* * * * * root /home/nodejs/cdn-purge.sh' + - '* */4 * * * nodejs rsync -aqz --delete benchmark:charts/ /home/www/benchmarking/charts/' + - '* */4 * * * nodejs rsync -aqz --delete benchmark:coverage-out/out/ /home/www/coverage/' + tags: setup diff --git a/setup/www/tasks/tools.yaml b/setup/www/tasks/tools.yaml new file mode 100644 index 000000000..77e43475b --- /dev/null +++ b/setup/www/tasks/tools.yaml @@ -0,0 +1,30 @@ +- name: dist-indexer | Install dist-indexer + command: "npm install nodejs-dist-indexer -g" + tags: dist-indexer + +- name: latest-linker | Install latest-linker + command: "npm install nodejs-latest-linker -g" + tags: latest-linker + +- name: nightly-builder | Install nightly-builder + command: "npm install nodejs-nightly-builder -g" + tags: nightly-builder + +- name: nightly-builder | Copy config + template: + src: ./resources/config/nightly-builder.json.j2 + dest: /etc/nightly-builder.json + mode: 0644 + tags: nightly-builder + + #TODO: the branches are hardwired here, they should be in vars somewhere +- name: tools | Add periodic tasks to crontab + lineinfile: + dest: /etc/crontab + line: "{{ item }}" + with_items: + - '0 20 * * * dist /usr/bin/nodejs-nightly-builder --type nightly --ref heads/v6.x --config /etc/nightly-builder.json' + - '0 19 * * * dist /usr/bin/nodejs-nightly-builder --type nightly --ref heads/v7.x --config /etc/nightly-builder.json' + - '0 18 * * * dist /usr/bin/nodejs-nightly-builder --type nightly --ref heads/master --config /etc/nightly-builder.json' + - '1 0 * * * root npm update -g nodejs-latest-linker nodejs-dist-indexer nodejs-nightly-builder >& /dev/null' + tags: tools diff --git a/setup/www/tasks/user.yaml b/setup/www/tasks/user.yaml index bd9a31878..a57fa9745 100644 --- a/setup/www/tasks/user.yaml +++ b/setup/www/tasks/user.yaml @@ -13,14 +13,6 @@ command: usermod -aG staging dist tags: user -- name: User | Download pubkeys (root_users) - get_url: - url: https://github.com/{{ item }}.keys - dest: /tmp/{{ item }}.keys - delegate_to: 127.0.0.1 - with_items: root_users - tags: user - - name: User | Download pubkeys (dist_users) get_url: url: https://github.com/{{ item }}.keys @@ -37,27 +29,6 @@ with_items: libuv_users tags: user -- name: User | Create authorized_keys for root - authorized_key: - user: "root" - key: "{{ lookup('file', '/tmp/' + item + '.keys') }}" - with_items: root_users - tags: user - -- name: User | Create authorized_keys for nodejs - authorized_key: - user: "nodejs" - key: "{{ lookup('file', '/tmp/' + item + '.keys') }}" - with_items: root_users - tags: user - -- name: User | Create authorized_keys for dist (root_users) - authorized_key: - user: "dist" - key: "{{ lookup('file', '/tmp/' + item + '.keys') }}" - with_items: root_users - tags: user - - name: User | Create authorized_keys for dist (dist_users) authorized_key: user: "dist" diff --git a/setup/www/tasks/webhook.yaml b/setup/www/tasks/webhook.yaml index f06c68e04..21d4546eb 100644 --- a/setup/www/tasks/webhook.yaml +++ b/setup/www/tasks/webhook.yaml @@ -3,26 +3,21 @@ tags: webhook - name: GitHub Webhook | Copy config - copy: - src: ./resources/config/github-webhook.json + template: + src: ./resources/config/github-webhook.json.j2 dest: /etc/github-webhook.json mode: 0644 tags: webhook -- name: GitHub Webhook | Copy secret to config - replace: - dest: /etc/github-webhook.json - regexp: '\{\{github_secret\}\}' - replace: '{{ github_secret }}' - tags: webhook - -- name: GitHub Webhook | Copy Upstart config - copy: - src: ./resources/config/github-webhook.conf - dest: /etc/init/github-webhook.conf - mode: 0644 +- name: GitHub Webhook | Generate and copy init script + template: + src: ./resources/config/github-webhook.service.j2 + dest: /lib/systemd/system/github-webhook.service tags: webhook - name: GitHub Webhook | Start service - service: name=github-webhook state=started - tags: webhook \ No newline at end of file + service: + name: github-webhook + state: restarted + enabled: yes + tags: webhook diff --git a/setup/www/tools/dist-indexer/LICENSE b/setup/www/tools/dist-indexer/LICENSE deleted file mode 100644 index 843e13c23..000000000 --- a/setup/www/tools/dist-indexer/LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright io.js contributors. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to -deal in the Software without restriction, including without limitation the -rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -sell copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. diff --git a/setup/www/tools/dist-indexer/README.md b/setup/www/tools/dist-indexer/README.md deleted file mode 100644 index 7b65e6fe8..000000000 --- a/setup/www/tools/dist-indexer/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# dist-indexer - -**An application to create io.js distribution index files: index.json and index.tab** - -Output can be seen: - - * https://iojs.org/download/nightly/index.tab - * https://iojs.org/download/nightly/index.json - -## Usage: - -```text -dist-indexer.js \ - --dist /path/to/dist/directory/ - --indexjson /path/to/dist/directory/index.json - --indextab /path/to/dist/directory/index.tab -``` - ------------------------------------ - -Copyright io.js contributors. All rights reserved. - -Licensed under MIT, see the LICENSE file for details diff --git a/setup/www/tools/dist-indexer/decode-ref.js b/setup/www/tools/dist-indexer/decode-ref.js deleted file mode 100644 index de41994b2..000000000 --- a/setup/www/tools/dist-indexer/decode-ref.js +++ /dev/null @@ -1,31 +0,0 @@ -const assert = require('assert') - , dirre = /^(v\d+\.\d+\.\d+(?:-rc\.\d+)?)(?:-(?:(?:next-)?nightly|test)\d{8}(\w+))?$/ // get version or commit from dir name - - -function decodeRef (dir) { - var m = dir.match(dirre) - return m && (m[2] || m[1]) -} - - -module.exports = decodeRef - - -if (module === require.main) { - var tests = [ - { dir: 'v1.0.0' , ref: 'v1.0.0' } - , { dir: 'v10.11.12' , ref: 'v10.11.12' } - , { dir: 'v2.3.2-nightly20150625dcbb9e1da6' , ref: 'dcbb9e1da6' } - , { dir: 'v2.3.1-next-nightly201506308f6f4280c6' , ref: '8f6f4280c6' } - , { dir: 'v3.0.0-rc.1' , ref: 'v3.0.0-rc.1' } - , { dir: 'v33.22.1-rc.111' , ref: 'v33.22.1-rc.111' } - , { dir: 'v0.6.1' , ref: 'v0.6.1' } - , { dir: 'v0.5.1' , ref: 'v0.5.1' } - , { dir: 'v6.0.0-test20151107093b0e865c' , ref: '093b0e865c' } - ] - - tests.forEach(function (test) { - console.log(`testing ${test.dir} -> ${test.ref}`) - assert.equal(decodeRef(test.dir), test.ref) - }) -} diff --git a/setup/www/tools/dist-indexer/dist-indexer.js b/setup/www/tools/dist-indexer/dist-indexer.js deleted file mode 100755 index 0916e0528..000000000 --- a/setup/www/tools/dist-indexer/dist-indexer.js +++ /dev/null @@ -1,537 +0,0 @@ -#!/usr/bin/env node - -'use strict'; - -const fs = require('fs') - , path = require('path') - , argv = require('minimist')(process.argv.slice(2)) - , map = require('map-async') - , after = require('after') - , hyperquest = require('hyperquest') - , bl = require('bl') - , semver = require('semver') - - , transformFilename = require('./transform-filename') - , decodeRef = require('./decode-ref') - - , versionCachePath = path.join(process.env.HOME, '.dist-indexer-version-cache') - - // needs auth: githubContentUrl = 'https://api.github.com/repos/nodejs/node/contents' - , githubContentUrl = 'https://raw.githubusercontent.com/nodejs/{repo}/{gitref}' - , npmPkgJsonUrl = `${githubContentUrl}/deps/npm/package.json` - , v8VersionUrl = [ - `${githubContentUrl}/deps/v8/src/version.cc` - , `${githubContentUrl}/deps/v8/include/v8-version.h` - ] - , uvVersionUrl = [ - `${githubContentUrl}/deps/uv/include/uv-version.h` - , `${githubContentUrl}/deps/uv/src/version.c` - , `${githubContentUrl}/deps/uv/include/uv.h` - ] - , sslVersionUrl = [ - `${githubContentUrl}/deps/openssl/openssl/include/openssl/opensslv.h` - , `${githubContentUrl}/deps/openssl/openssl/Makefile` - ] - , zlibVersionUrl = `${githubContentUrl}/deps/zlib/zlib.h` - , modVersionUrl = [ - `${githubContentUrl}/src/node_version.h` - , `${githubContentUrl}/src/node.h` - ] - , ltsVersionUrl = `${githubContentUrl}/src/node_version.h` - , githubOptions = { headers: { - 'accept': 'text/plain,application/vnd.github.v3.raw' - } } - - -if (typeof argv.dist != 'string') - throw new Error('Missing --dist argument') - -if (typeof argv.indexjson != 'string') - throw new Error('Missing --indexjson argument') - -if (typeof argv.indextab != 'string') - throw new Error('Missing --indextab argument') - -if (!fs.statSync(argv.dist).isDirectory()) - throw new Error('"%s" is not a directory') - - -var versionCache = {} - -try { - versionCache = JSON.parse(fs.readFileSync(versionCachePath, 'utf8')) -} catch (e) {} - - -function saveVersionCache () { - fs.writeFileSync(versionCachePath, JSON.stringify(versionCache), 'utf8') -} - - -function cacheGet (gitref, prop) { - return versionCache[gitref] && versionCache[gitref][prop] -} - - -function cachePut (gitref, prop, value) { - if (prop && (value || value === false)) { - (versionCache[gitref] || (versionCache[gitref] = {}))[prop] = value - } -} - - -function fetch (url, gitref, callback) { - let repo = (/^v0\.\d\./).test(gitref) - ? 'node-v0.x-archive' - : 'node' - url = url.replace('{gitref}', gitref) - .replace('{repo}', repo) - + `?rev=${gitref}` - hyperquest.get(url, githubOptions).pipe(bl(function (err, data) { - if (err) - return callback(err) - - callback(null, data.toString()) - })) -} - - -function fetchNpmVersion (gitref, callback) { - var version = cacheGet(gitref, 'npm') - if (version || (/^v0\.([012345]\.\d+|6\.[0-2])$/).test(gitref)) - return setImmediate(callback.bind(null, null, version)) - - fetch(npmPkgJsonUrl, gitref, function (err, rawData) { - if (err) - return callback(err) - - var data - - try { - data = JSON.parse(rawData) - } catch (e) { - return callback(e) - } - - cachePut(gitref, 'npm', data.version) - return callback(null, data.version) - }) -} - - -function fetchV8Version (gitref, callback) { - var version = cacheGet(gitref, 'v8') - if (version) - return setImmediate(callback.bind(null, null, version)) - - fetch(v8VersionUrl[0], gitref, function (err, rawData) { - if (err) - return callback(err) - - version = rawData.split('\n').map(function (line) { - return line.match(/^#define (?:MAJOR_VERSION|MINOR_VERSION|BUILD_NUMBER|PATCH_LEVEL)\s+(\d+)$/) - }) - .filter(Boolean) - .map(function (m) { return m[1] }) - .join('.') - - if (version) { - cachePut(gitref, 'v8', version) - return callback(null, version) - } - - fetch(v8VersionUrl[1], gitref, function (err, rawData) { - if (err) - return callback(err) - - version = rawData.split('\n').map(function (line) { - return line.match(/^#define V8_(?:MAJOR_VERSION|MINOR_VERSION|BUILD_NUMBER|PATCH_LEVEL)\s+(\d+)$/) - }) - .filter(Boolean) - .map(function (m) { return m[1] }) - .join('.') - - cachePut(gitref, 'v8', version) - callback(null, version) - }) - }) -} - - -function fetchUvVersion (gitref, callback) { - var version = cacheGet(gitref, 'uv') - if (version || (/^v0\.([01234]\.\d+|5\.0)$/).test(gitref)) - return setImmediate(callback.bind(null, null, version)) - - fetch(uvVersionUrl[0], gitref, function (err, rawData) { - if (err) - return callback(err) - - version = rawData.split('\n').map(function (line) { - return line.match(/^#define UV_VERSION_(?:MAJOR|MINOR|PATCH)\s+(\d+)$/) - }) - .filter(Boolean) - .map(function (m) { return m[1] }) - .join('.') - - if (version) { - cachePut(gitref, 'uv', version) - return callback(null, version) - } - - fetch(uvVersionUrl[1], gitref, function (err, rawData) { - if (err) - return callback(err) - - version = rawData.split('\n').map(function (line) { - return line.match(/^#define UV_VERSION_(?:MAJOR|MINOR|PATCH)\s+(\d+)$/) - }) - .filter(Boolean) - .map(function (m) { return m[1] }) - .join('.') - - if (version) { - cachePut(gitref, 'uv', version) - return callback(null, version) - } - - fetch(uvVersionUrl[2], gitref, function (err, rawData) { - if (err) - return callback(err) - - version = rawData.split('\n').map(function (line) { - return line.match(/^#define UV_VERSION_(?:MAJOR|MINOR|PATCH)\s+(\d+)$/) - }) - .filter(Boolean) - .map(function (m) { return m[1] }) - .join('.') - - cachePut(gitref, 'uv', version) - callback(null, version) - }) - }) - }) -} - - -function fetchSslVersion (gitref, callback) { - var version = cacheGet(gitref, 'ssl') - if (version || (/^v0\.([01234]\.\d+|5\.[0-4])$/).test(gitref)) - return setImmediate(callback.bind(null, null, version)) - - fetch(sslVersionUrl[0], gitref, function (err, rawData) { - if (err) - return callback(err) - - var m = rawData.match(/^#\s*define OPENSSL_VERSION_TEXT\s+"OpenSSL ([^\s]+)/m) - version = m && m[1] - - if (version) { - version = version.replace(/-fips$/, '') - cachePut(gitref, 'ssl', version) - - return callback(null, version) - } - - fetch(sslVersionUrl[1], gitref, function (err, rawData) { - if (err) - return callback(err) - - var m = rawData.match(/^VERSION=(.+)$/m) - version = m && m[1] - cachePut(gitref, 'ssl', version) - - callback(null, version) - }) - }) -} - - -function fetchZlibVersion (gitref, callback) { - var version = cacheGet(gitref, 'zlib') - if (version || (/^v0\.([01234]\.\d+|5\.[0-7])$/).test(gitref)) - return setImmediate(callback.bind(null, null, version)) - - fetch(zlibVersionUrl, gitref, function (err, rawData) { - if (err) - return callback(err) - - var m = rawData.match(/^#define ZLIB_VERSION\s+"(.+)"$/m) - version = m && m[1] - cachePut(gitref, 'zlib', version) - - callback(null, version) - }) -} - - -function fetchModVersion (gitref, callback) { - var version = cacheGet(gitref, 'mod') - if (version || (/^v0\.1\.\d+$/).test(gitref)) - return setImmediate(callback.bind(null, null, version)) - - fetch(modVersionUrl[0], gitref, function (err, rawData) { - if (err) - return callback(err) - - var m = rawData.match(/^#define NODE_MODULE_VERSION\s+([^\s]+)\s+.+$/m) - version = m && m[1] - - if (version) { - cachePut(gitref, 'mod', version) - return callback(null, version) - } - - fetch(modVersionUrl[1], gitref, function (err, rawData) { - if (err) - return callback(err) - - m = rawData.match(/^#define NODE_MODULE_VERSION\s+\(?([^\s\)]+)\)?\s+.+$/m) - version = m && m[1] - cachePut(gitref, 'mod', version) - callback(null, version) - }) - }) -} - - -function fetchLtsVersion (gitref, callback) { - var version = cacheGet(gitref, 'lts') - - if (version || version === false) - return setImmediate(callback.bind(null, null, version)) - - fetch(ltsVersionUrl, gitref, function (err, rawData) { - if (err) - return callback(err) - - var m = rawData.match(/^#define NODE_VERSION_IS_LTS 1$/m) - if (m) { - m = rawData.match(/^#define NODE_VERSION_LTS_CODENAME "([^"]+)"$/m) - version = m && m[1] - } else - version = false - - cachePut(gitref, 'lts', version) - callback(null, version) - }) -} - - -function dirDate (dir, callback) { - fs.readdir(path.join(argv.dist, dir), function (err, files) { - if (err) - return callback(err) - - map(files, mtime, afterMap) - - function mtime (file, callback) { - fs.stat(path.join(argv.dist, dir, file), function (err, stat) { - callback(null, stat && stat.mtime) - }) - } - - function afterMap (err, mtimes) { - mtimes = mtimes.filter(Boolean) - mtimes.sort((d1, d2) => d1 < d2 ? -1 : d1 > d2 ? 1 : 0) - callback(null, mtimes[0]) - } - }) -} - - -function dirFiles (dir, callback) { - //TODO: look in SHASUMS.txt as well for older versions - fs.readFile(path.join(argv.dist, dir, 'SHASUMS256.txt'), 'utf8', afterReadFile) - - function afterReadFile (err, contents) { - if (err) - return callback(err) - - var files = contents.split('\n').map(function (line) { - var seg = line.split(/\s+/) - return seg.length >= 2 && seg[1] - }) - .map(transformFilename) - .filter(Boolean) - .sort() - - callback(null, files) - } -} - - -function inspectDir (dir, callback) { - var gitref = decodeRef(dir) - , files - , npmVersion - , v8Version - , uvVersion - , sslVersion - , zlibVersion - , modVersion - , ltsVersion - , date - - if (!gitref) { - return fs.stat(path.join(argv.dist, dir), function (err, stat) { - if (err) - return callback(err) - if (stat.isDirectory() && !(/^(latest|npm$|patch$|v0\.10\.16-isaacs-manual$)/).test(dir)) - console.error(`Ignoring directory "${dir}" (can't decode ref)`) - return callback() - }) - } - - dirFiles(dir, function (err, _files) { - if (err) { - console.error(`Ignoring directory "${dir}" (can't decode dir contents)`) - return callback() // not a dir we care about - } - - files = _files - - var done = after(8, afterAll) - - dirDate(dir, function (err, _date) { - if (err) - return done(err) - - date = _date - done() - }) - - fetchNpmVersion(gitref, function (err, version) { - if (err) { - console.error(err) - console.error('(ignoring error fetching npm version for %s)', gitref) - } - npmVersion = version - done() - }) - - fetchV8Version(gitref, function (err, version) { - if (err) { - console.error(err) - console.error('(ignoring error fetching V8 version for %s)', gitref) - } - v8Version = version - done() - }) - - fetchUvVersion(gitref, function (err, version) { - if (err) { - console.error(err) - console.error('(ignoring error fetching uv version for %s)', gitref) - } - uvVersion = version - done() - }) - - fetchSslVersion(gitref, function (err, version) { - if (err) { - console.error(err) - console.error('(ignoring error fetching OpenSSL version for %s)', gitref) - } - sslVersion = version - done() - }) - - fetchZlibVersion(gitref, function (err, version) { - if (err) { - console.error(err) - console.error('(ignoring error fetching zlib version for %s)', gitref) - } - zlibVersion = version - done() - }) - - fetchModVersion(gitref, function (err, version) { - if (err) { - console.error(err) - console.error('(ignoring error fetching modules version for %s)', gitref) - } - modVersion = version - done() - }) - - fetchLtsVersion(gitref, function (err, version) { - if (err) { - console.error(err) - console.error('(ignoring error fetching LTS version for %s)', gitref) - } - ltsVersion = version - done() - }) - }) - - function afterAll (err) { - if (err) { - console.error(err) - console.error('(ignoring directory due to error %s)', dir) - return callback() - } - - callback(null, { - version : dir - , date : date.toISOString().substring(0, 10) - , files : files - , npm : npmVersion - , v8 : v8Version - , uv : uvVersion - , zlib : zlibVersion - , openssl : sslVersion - , modules : modVersion - , lts : ltsVersion - }) - } -} - - -map(fs.readdirSync(argv.dist).sort().reverse(), inspectDir, afterMap) - - -function afterMap (err, dirs) { - if (err) - throw err - - dirs.sort(function (d1, d2) { - return semver.compare(d2.version, d1.version) - }) - - saveVersionCache() - - dirs = dirs.filter(Boolean) - - var jsonOut = fs.createWriteStream(argv.indexjson, 'utf8') - , tabOut = fs.createWriteStream(argv.indextab, 'utf8') - - function tabWrite () { - var args = [].slice.call(arguments).map((a) => a || '-') - tabOut.write(args.join('\t') + '\n') - } - - jsonOut.write('[\n') - tabWrite('version', 'date', 'files', 'npm', 'v8', 'uv', 'zlib', 'openssl', 'modules', 'lts') - - dirs.forEach(function (dir, i) { - jsonOut.write(JSON.stringify(dir) + (i != dirs.length - 1 ? ',\n' : '\n')) - tabWrite( - dir.version - , dir.date - , dir.files.join(',') - , dir.npm - , dir.v8 - , dir.uv - , dir.zlib - , dir.openssl - , dir.modules - , dir.lts - ) - }) - - jsonOut.write(']\n') - - jsonOut.end() - tabOut.end() -} diff --git a/setup/www/tools/dist-indexer/ls-types.js b/setup/www/tools/dist-indexer/ls-types.js deleted file mode 100755 index 62a93544b..000000000 --- a/setup/www/tools/dist-indexer/ls-types.js +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env node - -const fs = require('fs') - , path = require('path') - - , transformFilename = require('./transform-filename') - - -function lsTypes (dir, dones, callback) { - if (typeof dones == 'function') { - callback = dones - dones = false - } - - fs.readdir(dir, afterReaddir) - - function afterReaddir (err, files) { - if (err) - return callback(err) - - if (dones) { - files = files.map(function (f) { - if (/\.done$/.test(f)) - return f.replace(/\.done$/, '') - return false - }).filter(Boolean) - } - - files = files.map(transformFilename) - .filter(Boolean) - .sort() - - callback(null, files) - } -} - - -if (require.main === module) { - var dones = process.argv[2] == '-d' - , dirs = process.argv.slice(dones ? 3 : 2) - - ;(function ls () { - var dir = dirs.shift() - if (!dir) - return - - fs.stat(dir, function (err, stat) { - if (err || !stat.isDirectory()) - return ls() - - lsTypes(dir, dones, function (err, files) { - if (err) - throw err - - files.length && console.log('%s: %s', dir, files.join(' ')) - ls() - }) - }) - }()) -} diff --git a/setup/www/tools/dist-indexer/package.json b/setup/www/tools/dist-indexer/package.json deleted file mode 100644 index f2b8a3202..000000000 --- a/setup/www/tools/dist-indexer/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "dist-indexer", - "version": "1.0.0", - "description": "", - "main": "dist-indexer.js", - "author": "Rod Vagg ", - "license": "MIT", - "dependencies": { - "after": "~0.8.1", - "bl": "~0.9.3", - "hyperquest": "~1.0.1", - "map-async": "~0.1.1", - "minimist": "~1.1.0", - "semver": "~5.0.1" - } -} diff --git a/setup/www/tools/dist-indexer/transform-filename.js b/setup/www/tools/dist-indexer/transform-filename.js deleted file mode 100644 index 83eeb7056..000000000 --- a/setup/www/tools/dist-indexer/transform-filename.js +++ /dev/null @@ -1,128 +0,0 @@ -const assert = require('assert') - , types = { - 'tar.gz' : 'src' - , 'aix-ppc64' : 'aix-ppc64' - , 'darwin-x64' : 'osx-x64-tar' - , 'darwin-x86' : 'osx-x86-tar' - , 'pkg' : 'osx-x64-pkg' - , 'linux-arm64' : 'linux-arm64' - , 'linux-armv7l' : 'linux-armv7l' - , 'linux-armv6l' : 'linux-armv6l' - , 'linux-x64' : 'linux-x64' - , 'linux-x86' : 'linux-x86' - , 'linux-ppc64le' : 'linux-ppc64le' - , 'sunos-x64' : 'sunos-x64' - , 'sunos-x86' : 'sunos-x86' - , 'x64.msi' : 'win-x64-msi' - , 'x86.msi' : 'win-x86-msi' - , 'win-x64/iojs.exe' : 'win-x64-exe' - , 'win-x86/iojs.exe' : 'win-x86-exe' - , 'node.exe' : 'win-x86-exe' - , 'x64/node.exe' : 'win-x64-exe' - , 'headers' : 'headers' - } - - -function transformFilename (file) { - file = file && file.replace(/^(?:\.\/)?(?:iojs|node)-v\d+\.\d+\.\d+-?((rc\.\d+|(next-)?nightly\d{8}[^-\.]+)-?)?\.?/, '') - .replace(/\.tar\.gz$/, '') - return types[file] -} - - -module.exports = transformFilename - - -if (module === require.main) { - var tests = [ - { file: 'doc' } - , { file: 'node-v6.7.0-aix-ppc64.tar.gz', type: 'aix-ppc64'} - , { file: 'iojs-v3.0.0-darwin-x64.tar.gz', type: 'osx-x64-tar' } - , { file: 'iojs-v3.0.0-darwin-x64.tar.xz' } - , { file: 'iojs-v3.0.0-headers.tar.gz', type: 'headers' } - , { file: 'iojs-v3.0.0-headers.tar.xz' } - , { file: 'iojs-v3.0.0-linux-armv7l.tar.gz', type: 'linux-armv7l' } - , { file: 'iojs-v3.0.0-linux-armv7l.tar.xz' } - , { file: 'iojs-v3.0.0-linux-armv6l.tar.gz', type: 'linux-armv6l' } - , { file: 'iojs-v3.0.0-linux-armv6l.tar.xz' } - , { file: 'iojs-v3.0.0-linux-x64.tar.gz', type: 'linux-x64' } - , { file: 'iojs-v3.0.0-linux-x64.tar.xz' } - , { file: 'iojs-v3.0.0-linux-x86.tar.gz', type: 'linux-x86' } - , { file: 'iojs-v3.0.0-linux-x86.tar.xz' } - , { file: 'iojs-v3.0.0.pkg', type: 'osx-x64-pkg' } - , { file: 'iojs-v3.0.0.tar.gz', type: 'src' } - , { file: 'iojs-v3.0.0.tar.xz' } - , { file: 'iojs-v3.0.0-x64.msi', type: 'win-x64-msi' } - , { file: 'iojs-v3.0.0-x86.msi', type: 'win-x86-msi' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-darwin-x64.tar.gz', type: 'osx-x64-tar' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-darwin-x64.tar.xz' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-headers.tar.gz', type: 'headers' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-headers.tar.xz' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-linux-armv7l.tar.gz', type: 'linux-armv7l' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-linux-armv7l.tar.xz' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-linux-armv6l.tar.gz', type: 'linux-armv6l' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-linux-armv6l.tar.xz' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-linux-x64.tar.gz', type: 'linux-x64' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-linux-x64.tar.xz' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-linux-x86.tar.gz', type: 'linux-x86' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-linux-x86.tar.xz' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6.pkg', type: 'osx-x64-pkg' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6.tar.gz', type: 'src' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6.tar.xz' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-x64.msi', type: 'win-x64-msi' } - , { file: 'iojs-v3.0.0-nightly20150625dcbb9e1da6-x86.msi', type: 'win-x86-msi' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-darwin-x64.tar.gz', type: 'osx-x64-tar' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-darwin-x64.tar.xz' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-headers.tar.gz', type: 'headers' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-headers.tar.xz' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-linux-armv7l.tar.gz', type: 'linux-armv7l' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-linux-armv7l.tar.xz' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-linux-armv6l.tar.gz', type: 'linux-armv6l' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-linux-armv6l.tar.xz' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-linux-x64.tar.gz', type: 'linux-x64' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-linux-x64.tar.xz' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-linux-x86.tar.gz', type: 'linux-x86' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-linux-x86.tar.xz' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6.pkg', type: 'osx-x64-pkg' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6.tar.gz', type: 'src' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6.tar.xz' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-x64.msi', type: 'win-x64-msi' } - , { file: 'iojs-v3.0.0-next-nightly20150625dcbb9e1da6-x86.msi', type: 'win-x86-msi' } - , { file: 'iojs-v3.0.0-rc.1-darwin-x64.tar.gz', type: 'osx-x64-tar' } - , { file: 'iojs-v3.0.0-rc.1-darwin-x64.tar.xz' } - , { file: 'iojs-v3.0.0-rc.1-headers.tar.gz', type: 'headers' } - , { file: 'iojs-v3.0.0-rc.1-headers.tar.xz' } - , { file: 'iojs-v3.0.0-rc.1-linux-armv7l.tar.gz', type: 'linux-armv7l' } - , { file: 'iojs-v3.0.0-rc.1-linux-armv7l.tar.xz' } - , { file: 'iojs-v3.0.0-rc.1-linux-armv6l.tar.gz', type: 'linux-armv6l' } - , { file: 'iojs-v3.0.0-rc.1-linux-armv6l.tar.xz' } - , { file: 'iojs-v3.0.0-rc.1-linux-x64.tar.gz', type: 'linux-x64' } - , { file: 'iojs-v3.0.0-rc.1-linux-x64.tar.xz' } - , { file: 'iojs-v3.0.0-rc.1-linux-x86.tar.gz', type: 'linux-x86' } - , { file: 'iojs-v3.0.0-rc.1-linux-x86.tar.xz' } - , { file: 'iojs-v3.0.0-rc.1-linux-ppc64le.tar.gz', type: 'linux-ppc64le' } - , { file: 'iojs-v3.0.0-rc.1-linux-ppc64le.tar.xz' } - , { file: 'iojs-v3.0.0-rc.1.pkg', type: 'osx-x64-pkg' } - , { file: 'iojs-v3.0.0-rc.1.tar.gz', type: 'src' } - , { file: 'iojs-v3.0.0-rc.1.tar.xz' } - , { file: 'iojs-v3.0.0-rc.1-x64.msi', type: 'win-x64-msi' } - , { file: 'iojs-v3.0.0-rc.1-x86.msi', type: 'win-x86-msi' } - , { file: 'SHASUMS256.txt' } - , { file: 'win-x64/iojs.exe', type: 'win-x64-exe' } - , { file: 'win-x86/iojs.exe', type: 'win-x86-exe' } - , { file: 'node-v0.11.9-darwin-x64.tar.gz', type: 'osx-x64-tar' } - , { file: 'node-v0.11.9-darwin-x86.tar.gz', type: 'osx-x86-tar' } - , { file: 'node-v0.11.9-linux-x64.tar.gz', type: 'linux-x64' } - , { file: 'node-v0.11.9-linux-x86.tar.gz', type: 'linux-x86' } - , { file: 'node-v0.11.9-sunos-x64.tar.gz', type: 'sunos-x64' } - , { file: 'node-v0.11.9-sunos-x86.tar.gz', type: 'sunos-x86' } - , { file: 'node-v0.11.9.tar.gz', type: 'src' } - , { file: 'node.exe', type: 'win-x86-exe' } - , { file: 'x64/node.exe', type: 'win-x64-exe' } - ] - - tests.forEach(function (test) { - console.log(`testing ${test.file} -> ${test.type}`) - assert.equal(transformFilename(test.file), test.type) - }) -} diff --git a/setup/www/tools/dist/dist-promotable b/setup/www/tools/dist/dist-promotable index a3b0ee31b..0e55f14a3 100644 --- a/setup/www/tools/dist/dist-promotable +++ b/setup/www/tools/dist/dist-promotable @@ -9,4 +9,4 @@ if [ "X$site" != "Xiojs" ] && [ "X$site" != "Xnodejs" ]; then exit 1 fi -(cd /home/staging/${site}/release/ && /home/dist/tools/dist-indexer/ls-types.js -d *) +(cd /home/staging/${site}/release/ && nodejs-ls-types -d *) diff --git a/setup/www/tools/latest-linker/.gitignore b/setup/www/tools/latest-linker/.gitignore deleted file mode 100644 index b512c09d4..000000000 --- a/setup/www/tools/latest-linker/.gitignore +++ /dev/null @@ -1 +0,0 @@ -node_modules \ No newline at end of file diff --git a/setup/www/tools/latest-linker/latest-linker.js b/setup/www/tools/latest-linker/latest-linker.js deleted file mode 100755 index 34b5015c1..000000000 --- a/setup/www/tools/latest-linker/latest-linker.js +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env node - -'use strict' - -const fs = require('fs') - , path = require('path') - , semver = require('semver') - , map = require('map-async') - - , ltsNames = { 4: 'argon' } - -if (process.argv.length < 3) - throw new Error('Usage: latest-linker.js [docs directory]') - -const dir = path.resolve(process.argv[2]) - , docsDir = process.argv[3] && path.resolve(process.argv[3]) - -if (!fs.statSync(dir).isDirectory()) - throw new Error('Usage: latest-linker.js [docs directory]') - -if (docsDir && !fs.statSync(docsDir).isDirectory()) - throw new Error('Usage: latest-linker.js [docs directory]') - - -map( - fs.readdirSync(dir).map((d) => path.join(dir, d)) - , (d, callback) => fs.stat(d, (err, stat) => callback(null, { d: d, stat: stat })) - , afterMap -) - - -function afterMap (err, allDirs) { - if (err) - throw err - - allDirs = allDirs.filter((d) => d.stat && d.stat.isDirectory()) - .map((d) => path.basename(d.d)) - .map((d) => { try { return semver(d) } catch (e) {} }) - .filter(Boolean) - - makeDocsLinks(allDirs.map((v) => v.raw)) - - let dirs = allDirs.filter((d) => semver.satisfies(d, '~0.10 || ~0.12 || >= 1.0')) - .map((d) => d.raw) - - dirs.sort((d1, d2) => semver.compare(d1, d2)) - - link('0.10', dirs) - link(0.12, dirs) - - for (let i = 1;; i++) - if (!link(i, dirs) && i >= 4) break - - let max = link(null, dirs) - , tbreg = new RegExp(`(\\w+)-${max}.tar.gz`) - - let tarball = fs.readdirSync(path.join(dir, 'latest')) - .filter((f) => tbreg.test(f)) - - if (tarball.length != 1) - throw new Error('Could not find latest.tar.gz') - - tarball = tarball[0] - let name = tarball.match(tbreg)[1] - let dst = path.join(dir, `${name}-latest.tar.gz`) - try { fs.unlinkSync(dst) } catch (e) {} - fs.symlinkSync(path.join(dir, 'latest', tarball), dst) -} - - -function makeDocsLinks (versions) { - if (!docsDir) - return - - versions.forEach((version) => { - let src = path.join(dir, version, 'docs') - , dst = path.join(docsDir, version) - - fs.stat(src, (err, stat) => { - if (err || !stat.isDirectory()) - return - - fs.unlink(dst, () => { - fs.symlink(src, dst, (err) => { if (err) throw err }) - }) - }) - }) -} - - -function link (version, dirs) { - let line = version && `${version}.x` - , range = version ? `${Number(version) < 1 ? '~' : '^'}${line}` : '*' - , max = semver.maxSatisfying(dirs, range) - - if (!max) return false - - function symlink (name) { - let dst = path.join(dir, name) - , src = path.join(dir, max) - - try { fs.unlinkSync(dst) } catch (e) {} - fs.symlinkSync(src, dst) - - if (!docsDir) - return - - let dsrc = path.join(dir, max, 'docs') - , ddst = path.join(docsDir, name) - - try { fs.unlinkSync(ddst) } catch (e) {} - fs.symlinkSync(dsrc, ddst) - } - - if (line) { - symlink(`latest-v${line}`) - if (ltsNames[version]) - symlink(`latest-${ltsNames[version]}`) - } else { - symlink('latest') - } - - return max -} diff --git a/setup/www/tools/latest-linker/package.json b/setup/www/tools/latest-linker/package.json deleted file mode 100644 index 1d91915c5..000000000 --- a/setup/www/tools/latest-linker/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "latest-linker", - "version": "1.0.0", - "description": "", - "main": "latest-linker.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": "Rod (http://r.va.gg/)", - "license": "MIT", - "dependencies": { - "map-async": "~0.1.1", - "semver": "~5.0.1" - } -} diff --git a/setup/www/tools/metrics/download-counts.awk b/setup/www/tools/metrics/download-counts.awk index 4b55578e5..54b40e4f5 100644 --- a/setup/www/tools/metrics/download-counts.awk +++ b/setup/www/tools/metrics/download-counts.awk @@ -12,7 +12,7 @@ BEGIN { ($10 < 1000) { next } # unreasonably small download -!/\.(tar\.gz|tar\.xz|pkg|msi|exe)(\?[^ ]+)? HTTP\/1\.[10][" ]/ { +!/\.(tar\.gz|tar\.xz|pkg|msi|exe)(\?[^ ]+)? HTTP\/[12]\.[10][" ]/ { #print "Skipping:", $0 #check we're not missing anything with: grep Skipping /tmp/_var_log_nginx_nodejs.org-access.log | awk '{print $10 $8}'| grep -v '/$\|html$\|png$\|svg$\|json\|jpg$\|xml$\|txt$\|jar$\|js$\|pom$\|css$\|ico$\|zip$\|lib$\|exp$\|^40\|^30\|tab$\|eps$\|asc$\|gpg$\|pdf$\|tgz$\|\?\|\#\|pdb$\|rtf$\|md$\|SHASUMS' next @@ -24,7 +24,7 @@ BEGIN { { success = match($0, \ - / \[([^:]+).* "?GET (\/+(dist|download\/+release)\/+(node-latest\.tar\.gz|([^/]+)\/+((win-x64|win-x86|x64)?\/+?node\.exe|(x64\/)?node-+(v[0-9\.]+)[\-\.]([^\? ]+))))(\?[^ ]+)? HTTP\/1\.[01][" ]/ \ + / \[([^:]+).* "?GET (\/+(dist|download\/+release)\/+(node-latest\.tar\.gz|([^/]+)\/+((win-x64|win-x86|x64)?\/+?node\.exe|(x64\/)?node-+(v[0-9\.]+)[\-\.]([^\? ]+))))(\?[^ ]+)? HTTP\/[12]\.[01][" ]/ \ , m \ ) diff --git a/setup/www/tools/metrics/download-counts.sh b/setup/www/tools/metrics/download-counts.sh index 3b40d9b7f..5cf535696 100755 --- a/setup/www/tools/metrics/download-counts.sh +++ b/setup/www/tools/metrics/download-counts.sh @@ -63,7 +63,7 @@ function processsummaries { echo "Processing log files ..." -for i in $(ls -r /var/log/nginx/nodejs.org-access.log*); do +for i in $(ls -r /var/log/nginx/nodejs/nodejs.org-access.log*); do basename=$(basename $i | sed 's/\.[xg]z//') outfile=${logsoutputdir}/${basename}.csv summaryexists=true diff --git a/setup/www/tools/nightly-builder/.gitignore b/setup/www/tools/nightly-builder/.gitignore deleted file mode 100644 index 3c3629e64..000000000 --- a/setup/www/tools/nightly-builder/.gitignore +++ /dev/null @@ -1 +0,0 @@ -node_modules diff --git a/setup/www/tools/nightly-builder/build-required.js b/setup/www/tools/nightly-builder/build-required.js deleted file mode 100644 index 282a86cdc..000000000 --- a/setup/www/tools/nightly-builder/build-required.js +++ /dev/null @@ -1,49 +0,0 @@ -"use strict" - -const strftime = require('strftime').timezone(0) - , after = require('after') - , latestBuild = require('./latest-build') - , latestCommit = require('./latest-commit') - -const dateFormat = '%Y%m%d' - - -function timeString () { - return strftime(dateFormat) -} - - -function buildRequired (type, ref, force, config, callback) { - let done = after(2, onData) - , build - , commit - - function onData (err) { - if (err) - return callback(err) - - let buildCommit = build.commit.substr(0, 10) - , buildDate = strftime(dateFormat, build.date) - , nowDate = timeString() - - commit = commit.substr(0, 10) - - if (force || buildCommit != commit) // only need to compare commit - return callback(null, { type: type, commit: commit, date: nowDate }) - - return callback() - } - - latestBuild(type, function (err, data) { - build = data - done(err) - }) - - latestCommit(type, ref, config, function (err, data) { - commit = data - done(err) - }) -} - - -module.exports = buildRequired diff --git a/setup/www/tools/nightly-builder/latest-build.js b/setup/www/tools/nightly-builder/latest-build.js deleted file mode 100644 index 23c791bd7..000000000 --- a/setup/www/tools/nightly-builder/latest-build.js +++ /dev/null @@ -1,47 +0,0 @@ -"use strict" - -const jsonist = require('jsonist') - -const indexUrl = 'https://nodejs.org/download/{type}/index.json' - - -function latestBuild (type, callback) { - let url = indexUrl.replace(/\{type\}/, type) - - function onGet (err, data) { - if (err) - return callback(err) - - let i = -1 - - while (data[++i]) { - let m = data[i].version.match(/nightly(20\d\d)(\d\d)(\d\d)(\w{10,})/) - , date = new Date(m && `${m[1]}-${m[2]}-${m[3]}` || data[i].date) - , commit = m && m[4] - - if (m && commit) { - return callback(null, { - version : data[i].version - , date : date - , commit : commit - }) - } - } - - return callback(new Error(`no latest version for "${type}"`)) - } - - jsonist.get(url, onGet) -} - - -module.exports = latestBuild - -if (require.main == module) { - latestBuild('nightly', function (err, data) { - if (err) - throw err - - console.log('data', data) - }) -} diff --git a/setup/www/tools/nightly-builder/latest-commit.js b/setup/www/tools/nightly-builder/latest-commit.js deleted file mode 100644 index 48459aae9..000000000 --- a/setup/www/tools/nightly-builder/latest-commit.js +++ /dev/null @@ -1,26 +0,0 @@ -"use strict" - -const ghrepos = require('ghrepos') - - -function latestCommit (type, ref, config, callback) { - ghrepos.getRef( - { user: config.githubAuthUser, token: config.githubAuthToken } - , config.githubOrg - , config.githubRepo - , ref - , function (err, data) { - if (err) - return callback(err) - - if (!data || !data.object) - return callback(new Error(`Got unexpected data from GitHub: ${JSON.stringify(data)}`)) - - return callback(null, data.object.sha) - } - ) -} - - - -module.exports = latestCommit diff --git a/setup/www/tools/nightly-builder/nightly-builder.js b/setup/www/tools/nightly-builder/nightly-builder.js deleted file mode 100644 index c5f3cc6c3..000000000 --- a/setup/www/tools/nightly-builder/nightly-builder.js +++ /dev/null @@ -1,55 +0,0 @@ -"use strict" - -/* config file example: - -{ - "jenkinsToken": "X", - "jenkinsJobUrl": "https://ci-release.nodejs.org/job/iojs+release", - "githubAuthUser": "node-forward-bot", - "githubAuthToken": "X", - "githubOrg": "nodejs", - "githubRepo": "https://github.com/nodejs/node.git", - "githubScheme": "https://github.com/", - "releaseUrlBase": "https://nodejs.org/download/" -} - -*/ - -const argv = require('minimist')(process.argv.slice(2)) - , inspect = require('util').inspect - , xtend = require('xtend') - , buildRequired = require('./build-required') - , triggerBuild = require('./trigger-build') - , config = argv.config && require(argv.config) - -if (typeof argv.type != 'string' - || !/^(nightly|next-nightly)$/.test(argv.type) - || typeof argv.ref != 'string' - || !config - || typeof config.jenkinsToken != 'string' - || typeof config.jenkinsJobUrl != 'string' - || typeof config.githubAuthToken != 'string' - || typeof config.githubAuthUser != 'string' - || typeof config.githubRepo != 'string' - || typeof config.githubOrg != 'string' - || typeof config.githubScheme != 'string' - ) { - - console.error('Usage: nightly-builder --type --ref --config ') - return process.exit(1) -} - -buildRequired(argv.type, argv.ref, argv.force, config, function (err, data) { - if (err) - throw err - - if (!data) - return console.log('No build required') - - triggerBuild(xtend(data, config), function (err) { - if (err) - throw err - - console.log(`Build triggered: ${inspect(data)}`) - }) -}) diff --git a/setup/www/tools/nightly-builder/package.json b/setup/www/tools/nightly-builder/package.json deleted file mode 100644 index 3995c0e2e..000000000 --- a/setup/www/tools/nightly-builder/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "nightly-builder", - "version": "1.0.0", - "description": "nightly builder for io.js dist", - "main": "nightly-builder.js", - "author": "Rod Vagg ", - "license": "MIT", - "scripts": { - "test": "node ./test.js" - }, - "dependencies": { - "after": "~0.8.1", - "bl": "~0.9.4", - "ghrepos": "~1.0.2", - "hyperquest": "~1.0.1", - "jsonist": "~1.0.0", - "minimist": "~1.1.1", - "strftime": "~0.9.0", - "xtend": "~4.0.1" - }, - "devDependencies": { - "tape": "~4.0.0" - } -} diff --git a/setup/www/tools/nightly-builder/test.js b/setup/www/tools/nightly-builder/test.js deleted file mode 100644 index 9c30dd18a..000000000 --- a/setup/www/tools/nightly-builder/test.js +++ /dev/null @@ -1,47 +0,0 @@ -"use strict" - -const test = require('tape') - , latestCommit = require('./latest-commit') - , latestBuild = require('./latest-build') - - -test('latest-build', function (t) { - t.plan(14) - var other = null - - function verify (type) { - return function (err, data) { - t.error(err, 'no error') - t.ok(data, 'got data') - let m = data.version && data.version.match(/v\d+\.\d+\.\d+-((?:next-)?nightly)20\d{6}\w{10,}/) - t.ok(m, `version (${data.version}) looks correct`) - t.equal(m && m[1], type, 'correct type') - t.ok(data.date < new Date(), `date (${data.date.toISOString()}) looks correct`) - t.ok(data.commit && data.commit.length >= 10, `commit (${data.commit}) looks right`) - t.notEqual(data.commit, other, 'commit not the same as other type of commit') - other = data.commit // who's gonna be first?? - } - } - - latestBuild('nightly', verify('nightly')) - latestBuild('next-nightly', verify('next-nightly')) -}) - - -test('latest-commit', function (t) { - t.plan(8) - var other = null - - function verify (type) { - return function (err, sha) { - t.error(err, 'no error') - t.equal(typeof sha, 'string', 'got sha') - t.equal(sha.length, 40, `sha looks good (${sha})`) - t.notEqual(sha, other, 'sha not the same as other type of sha') - other = sha // who's gonna be first?? - } - } - - latestCommit('nightly', 'heads/v5.x', verify('nightly')) - latestCommit('next-nightly', 'heads/master', verify('next-nightly')) -}) diff --git a/setup/www/tools/nightly-builder/trigger-build.js b/setup/www/tools/nightly-builder/trigger-build.js deleted file mode 100644 index a4ce47227..000000000 --- a/setup/www/tools/nightly-builder/trigger-build.js +++ /dev/null @@ -1,75 +0,0 @@ -"use strict"; - -const hyperquest = require('hyperquest') - , jsonist = require('jsonist') - , bl = require('bl') - , qs = require('querystring') - - - -function triggerBuild(options, callback) { - let url = `${options.jenkinsJobUrl}/build?token=${options.jenkinsToken}` - , auth = `${options.githubAuthUser}:${options.githubAuthToken}` - , data = { - token : options.jenkinsToken - , parameter : [ - { - name : 'repository' - , value : `${options.githubScheme}${options.githubOrg}/${options.githubRepo}.git` - } - , { - name : 'commit' - , value : options.commit - } - , { - name : 'datestring' - , value : options.date - } - , { - name : 'disttype' - , value : options.type - } - , { - name : 'release_urlbase' - , value : `${options.releaseUrlBase}${options.type}/` - } - , { - name : 'rc' - , value : '0' - } - ] - } - - if (!options.jenkinsCrumbUrl) - return requestTrigger() - - jsonist.get(options.jenkinsCrumbUrl, { auth: auth }, function (err, data) { - if (err) - return callback(err) - - requestTrigger(data) - }) - - function requestTrigger (crumb) { - let postData = { - token : options.jenkinsToken - , json : JSON.stringify(data) - } - , post - , req - - postData[crumb.crumbRequestField] = crumb.crumb - post = qs.encode(postData) - req = hyperquest(url, { - method : 'post' - , headers : { 'content-type' : 'application/x-www-form-urlencoded' } - , auth : auth - }) - - req.end(post) - req.pipe(bl(callback)) - } -} - - -module.exports = triggerBuild diff --git a/setup/www/tools/promote/_promote.sh b/setup/www/tools/promote/_promote.sh index 577f3676b..98121a594 100755 --- a/setup/www/tools/promote/_promote.sh +++ b/setup/www/tools/promote/_promote.sh @@ -12,12 +12,17 @@ if [ -z ${dstdir+x} ]; then exit 1 fi +if [ -z ${dirmatch+x} ]; then + echo "\$dirmatch is not set" + exit 1 +fi + site=$1 version=$2 for subdir in $(cd $srcdir && ls); do - if [ -d "${srcdir}/${subdir}" ]; then + if [ -d "${srcdir}/${subdir}" ] && [[ $subdir =~ $dirmatch ]]; then resha=no if [ "X${version}" != "X" ] && [ "X${version}" != "X${subdir}" ]; then diff --git a/setup/www/tools/promote/_resha.sh b/setup/www/tools/promote/_resha.sh index b006b0d85..0ce9acb65 100755 --- a/setup/www/tools/promote/_resha.sh +++ b/setup/www/tools/promote/_resha.sh @@ -16,9 +16,12 @@ if [ "X${version}" == "X" ]; then exit 1 fi -(cd "${dstdir}/${version}" && shasum -a256 $(ls node* iojs* win-*/* 2> /dev/null) > SHASUMS256.txt) || exit 1 +(cd "${dstdir}/${version}" && shasum -a256 $(ls node* openssl* iojs* win-*/* x64/* 2> /dev/null) > SHASUMS256.txt) || exit 1 +if [[ $version =~ ^v[0] ]]; then + (cd "${dstdir}/${version}" && shasum $(ls node* openssl* x64/* 2> /dev/null) > SHASUMS.txt) || exit 1 +fi echo "${dstdir}/${version}/SHASUMS256.txt" -/home/dist/tools/dist-indexer/dist-indexer.js --dist $dstdir --indexjson ${dstdir}/index.json --indextab ${dstdir}/index.tab +nodejs-dist-indexer --dist $dstdir --indexjson ${dstdir}/index.json --indextab ${dstdir}/index.tab find "${dstdir}/${version}" -type f -exec chmod 644 '{}' \; find "${dstdir}/${version}" -type d -exec chmod 755 '{}' \; diff --git a/setup/www/tools/promote/promote_nightly.sh b/setup/www/tools/promote/promote_nightly.sh index aa1e2f7bf..c62369fe2 100755 --- a/setup/www/tools/promote/promote_nightly.sh +++ b/setup/www/tools/promote/promote_nightly.sh @@ -15,15 +15,24 @@ fi srcdir=$nightly_srcdir dstdir=$nightly_dstdir +dirmatch=$nightly_dirmatch . ${__dirname}/_promote.sh $site srcdir=$next_nightly_srcdir dstdir=$next_nightly_dstdir +dirmatch=$next_nightly_dirmatch . ${__dirname}/_promote.sh $site srcdir=$rc_srcdir dstdir=$rc_dstdir +dirmatch=$rc_dirmatch + +. ${__dirname}/_promote.sh $site + +srcdir=$test_srcdir +dstdir=$test_dstdir +dirmatch=$test_dirmatch . ${__dirname}/_promote.sh $site diff --git a/setup/www/tools/promote/promote_release.sh b/setup/www/tools/promote/promote_release.sh index d8d55d0ef..dca2e8eb0 100755 --- a/setup/www/tools/promote/promote_release.sh +++ b/setup/www/tools/promote/promote_release.sh @@ -20,7 +20,8 @@ fi srcdir=$release_srcdir dstdir=$release_dstdir +dirmatch=$release_dirmatch . ${__dirname}/_promote.sh $site $2 -/home/dist/tools/latest-linker/latest-linker.js /home/dist/${site}/release/ /home/dist/${site}/docs/ +nodejs-latest-linker /home/dist/${site}/release/ /home/dist/${site}/docs/ diff --git a/setup/www/tools/promote/settings b/setup/www/tools/promote/settings index 70dd3705b..b79562543 100755 --- a/setup/www/tools/promote/settings +++ b/setup/www/tools/promote/settings @@ -3,12 +3,20 @@ dist_rootdir=/home/dist/${site}/ release_srcdir=${staging_rootdir}release release_dstdir=${dist_rootdir}release +release_dirmatch=.* rc_srcdir=${staging_rootdir}custom rc_dstdir=${dist_rootdir}rc +rc_dirmatch="-rc\\." + +test_srcdir=${staging_rootdir}custom +test_dstdir=${dist_rootdir}test +test_dirmatch="-test[0-9]+" nightly_srcdir=${staging_rootdir}nightly nightly_dstdir=${dist_rootdir}nightly +nightly_dirmatch=.* next_nightly_srcdir=${staging_rootdir}next-nightly next_nightly_dstdir=${dist_rootdir}next-nightly +next_nightly_dirmatch=.*