基于kcat将TPC-DS数据快速导入kafka集群

# 安装依赖
sudo yum install gcc make flex bison byacc git

cd DSGen-software-code-3.2.0rc1/tools
make OS=LINUX

./dsdgen --help

dsdgen Population Generator (Version 3.2.0)
Copyright Transaction Processing Performance Council (TPC) 2001 - 2021

USAGE: dsdgen [options]

Note: When defined in a parameter file (using -p), parmeters should
use the form below. Each option can also be set from the command
line, using a form of '-param [optional argument]'
Unique anchored substrings of options are also recognized, and 
case is ignored, so '-sc' is equivalent to '-SCALE'

General Options
===============
ABREVIATION =  <s>       -- build table with abreviation <s>
DIR =  <s>               -- generate tables in directory <s>
HELP =  <n>              -- display this message
PARAMS =  <s>            -- read parameters from file <s>
QUIET =  [Y|N]           -- disable all output to stdout/stderr
SCALE =  <n>             -- volume of data to generate in GB
TABLE =  <s>             -- build only table <s>
UPDATE =  <n>            -- generate update data set <n>
VERBOSE =  [Y|N]         -- enable verbose output
PARALLEL =  <n>          -- build data in <n> separate chunks
CHILD =  <n>             -- generate <n>th chunk of the parallelized data
RELEASE =  [Y|N]         -- display the release information
_FILTER =  [Y|N]         -- output data to stdout
VALIDATE =  [Y|N]        -- produce rows for data validation

Advanced Options
===============
DELIMITER =  <s>         -- use <s> as output field separator
DISTRIBUTIONS =  <s>     -- read distributions from file <s>
FORCE =  [Y|N]           -- over-write data files without prompting
SUFFIX =  <s>            -- use <s> as output file suffix
TERMINATE =  [Y|N]       -- end each record with a field delimiter
VCOUNT =  <n>            -- set number of validation rows to be produced
VSUFFIX =  <s>           -- set file suffix for data validation
RNGSEED =  <n>           -- set RNG seed

mkdir ../data-100G

./dsdgen  -scale 100 -dir ../data/

awk '$0=NR"|"$0' store_sales.dat > store_sales_3.dat

yum install librdkafka-devel make gcc

git clone https://github.com/edenhill/kcat

cd kcat

./configure --prefix=/usr

make

make install

bin/kafka-topics.sh --bootstrap-server=host117:19092,host120:19092,host121:19092 \
 --create --partitions 6 --replication-factor 3 --topic store_sales_3_100g

 cat store_sales_3.dat | kcat \
-b host117:19092,host120:19092,host121:19092 \
-t store_sales_2_100g \
-P