update.py

#!/usr/bin/env python

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import os
import time


def modified_today(fname):
    now = time.time()
    try:
        return os.stat(fname).st_mtime > (now - now % (24 * 60 * 60))
    except OSError:
        return False


def call(cmd):
    print '+', cmd
    status = os.system(cmd)
    if status:
        raise Exception('invocation failed')


def main():
    call('time python make_db.py --buckets ../buckets.yaml --junit --threads 128')

    bq_cmd = 'bq load --source_format=NEWLINE_DELIMITED_JSON --max_bad_records=1000'
    mj_cmd = 'pypy make_json.py'

    mj_ext = ''
    bq_ext = ''
    if not modified_today('build_day.json.gz'):
        # cycle daily/weekly tables
        bq_ext = ' --replace'
        mj_ext = ' --reset-emitted'

    call(mj_cmd + mj_ext + ' --days 1 | pv | gzip > build_day.json.gz')
    call(bq_cmd + bq_ext + ' k8s-gubernator:build.day build_day.json.gz schema.json')

    call(mj_cmd + mj_ext + ' --days 7 | pv | gzip > build_week.json.gz')
    call(bq_cmd + bq_ext + ' k8s-gubernator:build.week build_week.json.gz schema.json')

    call(mj_cmd + ' | pv | gzip > build_all.json.gz')
    call(bq_cmd + ' k8s-gubernator:build.all build_all.json.gz schema.json')

    call('python stream.py --poll kubernetes-jenkins/gcs-changes/kettle '
         ' --dataset k8s-gubernator:build --tables all:0 day:1 week:7 --stop_at=1')


if __name__ == '__main__':
    os.chdir(os.path.dirname(__file__))
    os.environ['TZ'] = 'America/Los_Angeles'
    main()