google big_query 写数据

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Aug 13 15:23:34 2019

@author: lg
"""

import argparse
import csv
from google.cloud import bigquery


configs = {
#  'GOOGLE_APPLICATION_CREDENTIALS': '.cloud-test-199409-b79eed509dc5.json',
  'PROJECT': 'cloud-test-199409',
  'DATESET': 'ledi',
  
  'table':'t1'
}
    
def write_ltv_to_bq(filename, configs = configs):
    table = configs['table']
    
    client = bigquery.Client(project=configs['PROJECT'])
    dataset_id = configs['DATESET']

    dataset_ref = client.dataset(dataset_id)
    job_config = bigquery.LoadJobConfig()
    job_config.source_format = bigquery.SourceFormat.CSV
    # job_config.autodetect = True
    job_config.skip_leading_rows = 1
    job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND
    job_config.schema = [
        bigquery.SchemaField("player_id", "INTEGER"),
        bigquery.SchemaField("k", "INTEGER"),
#        bigquery.SchemaField("source", "STRING"),
#        bigquery.SchemaField("three_day_pay", "FLOAT"),
#        bigquery.SchemaField("seven_day_pay_predict", "FLOAT"),
#        bigquery.SchemaField("predict_time", "TIMESTAMP"),
#        bigquery.SchemaField("created_time", "TIMESTAMP"),
    ]
#    job_config.time_partitioning = bigquery.TimePartitioning(
#        type_ = bigquery.TimePartitioningType.DAY,
#        field = "created_time"
#    )
#    
    with open(filename, 'rb') as source_file:
        job = client.load_table_from_file(
            source_file,
            dataset_ref.table(table),
            location = 'US',
            job_config = job_config
        )

    job.result()


write_ltv_to_bq('lost1.csv')

向bigquery里写数据需要注意的是csv 文件的格式要和表的格式完全一致，即表的字段和csv 的字段必须完全一致，有三种方式向bigquery 里写数据：

• 从cloud Storage 里加载
• 本地csv 加载
• 将流数据加载

实话说，google的云计算非常垃圾，很不方便，google 的存储方式是列式存储，一般按照时间分片，感觉就是为了多赚钱，公司有５000万用户，我如果提取5000个用的数据，查询的费用和5000万一样多．
再多说两句

• 凡是上市的公司都是利益最大化为目标函数，即所有的上市公司在道德和理想与利益冲突时都会不要脸，所有公司
• google 死掉的会很快，用过google 的产品就知道了，阿里巴巴也会很快死掉
• 我用华为的手机和笔记本，华为的售后很好，华为在可预见的未来会很好，当华为上市的时候，就是华为灭亡的时候
• 当年牛逼哄哄的诺基亚，yahoo 都去了哪里