[Node.js] Fetch csv data and parse
2023-09-14 09:00:42 时间
We use needle
as a client on Node.js express server for fetching the data or site. https://www.npmjs.com/package/needle
The leanest and most handsome HTTP client in the Nodelands.
var needle = require('needle');
needle.get('http://www.google.com', function(error, response) {
if (!error && response.statusCode == 200)
console.log(response.body);
});
Callbacks not floating your boat? Needle got your back.
var data = {
file: '/home/johnlennon/walrus.png',
content_type: 'image/png'
};
// the callback is optional, and needle returns a `readableStream` object
// that triggers a 'done' event when the request/response process is complete.
needle
.post('https://my.server.com/foo', data, { multipart: true })
.on('readable', function() { /* eat your chunks */ })
.on('done', function(err) {
console.log('Ready-o!');
})
Then we use csv-parse
to parse the content: https://www.npmjs.com/package/csv-parse
import assert from 'assert';
import { parse } from 'csv-parse';
const records = [];
// Initialize the parser
const parser = parse({
delimiter: ':'
});
// Use the readable stream api to consume records
parser.on('readable', function(){
let record;
while ((record = parser.read()) !== null) {
records.push(record);
}
});
// Catch any error
parser.on('error', function(err){
console.error(err.message);
});
// Test that the parsed records matched the expected records
parser.on('end', function(){
assert.deepStrictEqual(
records,
[
[ 'root','x','0','0','root','/root','/bin/bash' ],
[ 'someone','x','1022','1022','','/home/someone','/bin/bash' ]
]
);
});
// Write data to the stream
parser.write("root:x:0:0:root:/root:/bin/bash\n");
parser.write("someone:x:1022:1022::/home/someone:/bin/bash\n");
// Close the readable stream
parser.end();
----
In this example, the pipe
function is used to stream the data from the remote CSV file to a local file, and the on
method is used to listen for the done
event, which is emitted once the file has finished downloading. The contents of the local file are then read using fs.readFileSync
and parsed using csv-parse
.
const needle = require('needle');
const parse = require('csv-parse');
const fs = require('fs');
needle.get('https://example.com/file.csv')
.pipe(fs.createWriteStream('file.csv'))
.on('done', function() {
// read the file once it's finished downloading
const csv = fs.readFileSync('file.csv', 'utf-8');
// parse the CSV content
const records = parse(csv, {
columns: true,
trim: true
});
// do something with the parsed data
console.log(records);
});
相关文章
- Node.js系列-http
- Node.js进程管理之子进程
- Node.js其他模块
- node.js原生后台进阶(一)
- [Node.js] Add node.js command line to global
- [CLI] Create a Hybrid Single-Multi Command Node.js CLI with Oclif and TypeScript
- [CLI] Create a Single-Command Node.js CLI with Oclif, TypeScript and Yarn Workspaces
- [Node.js] Using ES6 and beyond with Node.js
- [Node.js] node-persist: localStorage on the server
- [Node.js] Web Scraping Images with Node, Xray, and Download
- [Node.js] Intro to Web Scraping with Node and X-ray
- [Node.js] Using npm link to use node modules that are "in progress"
- Node.js刷新session过期时间
- [Node.js] mongoose schema
- [WASM] Run WebAssembly in Node.js using the node-loader
- [Node.js] Use Realm Object Database with Node.js
- [Node.js]25. Level 5. Route params
- [Node.js]23. Level 4: Semantic versioning
- [FE] nvm-windows: Microsoft/NPM/Google 推荐 Windows 的 Node.js 版本管理器
- 【nodejs原理&源码赏析(6)】深度剖析cluster模块源码与node.js多进程(下)
- Node.js v0.10.31API手冊-控制台