Skip to content
This repository has been archived by the owner on Jun 28, 2021. It is now read-only.

Commit

Permalink
info: new option
Browse files Browse the repository at this point in the history
  • Loading branch information
@wdavidw
wdavidw committed Nov 13, 2018
1 parent a72e0b1 commit 8c8132a
Show file tree
Hide file tree
Showing 6 changed files with 179 additions and 46 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.md
View file
Expand Up @@ -24,7 +24,7 @@ This is a complete rewrite based with a Buffer implementation. There are no majo

New features:

* new options from_line and to_line
* new options info, from_line and to_line
* trim: respect `ltrim` and `rtrim` when defined
* delimiter: may be a Buffer
* delimiter: handle multiple bytes/characters
Expand Down
83 changes: 56 additions & 27 deletions lib/es5/index.js
View file
Expand Up @@ -162,6 +162,13 @@ function (_Transform) {
throw new Error("Invalid Option Length: escape must be one character, got ".concat(options.escape.length));
} else {
options.escape = options.escape[0];
} // Normalize option `info`


if (options.info === undefined || options.info === null || options.info === false) {
options.info = false;
} else if (options.info !== true) {
throw new Error("Invalid Option: info must be true, got ".concat(JSON.stringify(options.info)));
} // Normalize option `quote`


Expand Down Expand Up @@ -217,6 +224,13 @@ function (_Transform) {
options.rtrim = false;
}

_this.info = {
comment_lines: 0,
empty_lines: 0,
invalid_field_length: 0,
lines: 1,
records: 0
};
_this.options = options;
_this.state = {
castField: fnCastField,
Expand All @@ -227,6 +241,7 @@ function (_Transform) {
expectedRecordLength: options.columns === null ? 0 : options.columns.length,
field: new ResizeableBuffer(20),
firstLineToHeaders: fnFirstLineToHeaders,
info: Object.assign({}, _this.info),
previousBuf: undefined,
quoting: false,
stop: false,
Expand All @@ -241,13 +256,6 @@ function (_Transform) {
wasQuoting: false,
wasRowDelimiter: false
};
_this.info = {
comment_lines: 0,
empty_lines: 0,
invalid_field_length: 0,
lines: 1,
records: 0
};
return _this;
}

Expand Down Expand Up @@ -285,6 +293,7 @@ function (_Transform) {
escape = _this$options.escape,
from = _this$options.from,
from_line = _this$options.from_line,
info = _this$options.info,
ltrim = _this$options.ltrim,
max_record_size = _this$options.max_record_size,
quote = _this$options.quote,
Expand Down Expand Up @@ -322,6 +331,12 @@ function (_Transform) {

if (this.state.wasRowDelimiter === true) {
this.info.lines++;
console.log(this.info.records, this.state.info.records);

if (info === true && this.state.record.length === 0 && this.state.field.length === 0 && this.state.wasQuoting === false) {
this.state.info = Object.assign({}, this.info);
}

this.state.wasRowDelimiter = false;
}

Expand All @@ -348,9 +363,7 @@ function (_Transform) {

if ((chr === cr || chr === nl) && this.state.wasRowDelimiter === false) {
this.state.wasRowDelimiter = true;
}

var recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); // Previous char was a valid escape char
} // Previous char was a valid escape char
// treat the current char as a regular char


Expand Down Expand Up @@ -420,6 +433,8 @@ function (_Transform) {
}

if (this.state.quoting === false) {
var recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos);

if (recordDelimiterLength !== 0) {
// Do not emit comments which take a full line
var skipCommentLine = this.state.commenting && this.state.record.length === 0 && this.state.field.length === 0;
Expand All @@ -434,7 +449,7 @@ function (_Transform) {
continue;
}

if (this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter ? 1 : 0) >= from_line) {
if (this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1 : 0) >= from_line) {
this.state.enabled = true;

this.__resetField();
Expand Down Expand Up @@ -491,8 +506,8 @@ function (_Transform) {
}
}

if (this.state.commenting === false && max_record_size !== 0) {
if (this.state.record_length + this.state.field.length > max_record_size) {
if (this.state.commenting === false) {
if (max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size) {
var _err2 = this.__error("Max Record Size: record exceed the maximum number of tolerated bytes of ".concat(max_record_size, " on line ").concat(this.info.lines));

if (_err2 !== undefined) return _err2;
Expand Down Expand Up @@ -550,6 +565,7 @@ function (_Transform) {
value: function __onRow() {
var _this$options2 = this.options,
columns = _this$options2.columns,
info = _this$options2.info,
from = _this$options2.from,
relax_column_count = _this$options2.relax_column_count,
raw = _this$options2.raw,
Expand Down Expand Up @@ -609,7 +625,7 @@ function (_Transform) {

if (from === 1 || this.info.records >= from) {
if (columns !== false) {
var obj = {};
var obj = {}; // Transform record array to an object

for (var i in record) {
if (columns[i].disabled) continue;
Expand All @@ -619,30 +635,39 @@ function (_Transform) {
var objname = this.options.objname;

if (objname === undefined) {
if (raw === true) {
this.push({
raw: this.state.rawBuffer.toString(),
if (raw === true || info === true) {
this.push(Object.assign({
record: obj
});
}, raw === true ? {
raw: this.state.rawBuffer.toString()
} : {}, info === true ? {
info: this.state.info
} : {}));
} else {
this.push(obj);
}
} else {
if (raw === true) {
this.push({
raw: this.state.rawBuffer.toString(),
if (raw === true || info === true) {
this.push(Object.assign({
record: [obj[objname], obj]
});
}, raw === true ? {
raw: this.state.rawBuffer.toString()
} : {}, info === true ? {
info: this.state.info
} : {}));
} else {
this.push([obj[objname], obj]);
}
}
} else {
if (raw === true) {
this.push({
raw: this.state.rawBuffer.toString(),
if (raw === true || info === true) {
this.push(Object.assign({
record: record
});
}, raw === true ? {
raw: this.state.rawBuffer.toString()
} : {}, info === true ? {
info: this.state.info
} : {}));
} else {
this.push(record);
}
Expand Down Expand Up @@ -675,12 +700,16 @@ function (_Transform) {
}, {
key: "__resetRow",
value: function __resetRow() {
var info = this.options.info;

if (this.options.raw === true) {
this.state.rawBuffer.reset();
}

this.state.record = [];
this.state.record_length = 0;
this.state.record_length = 0; // if(info === true){
// this.state.info = null
// }
}
}, {
key: "__onField",
Expand Down
5 changes: 5 additions & 0 deletions lib/index.d.ts
View file
Expand Up @@ -101,6 +101,11 @@ declare namespace parse {
* Start handling records from the requested line number.
*/
from_line?: number;

/**
* Generate two properties `info` and `record` where `info` is a snapshot of the info object at the time the record was created and `record` is the parsed array or object.
*/
info?: boolean;

/**
* If true, ignore whitespace immediately following the delimiter (i.e. left-trim all fields), defaults to false.
Expand Down
63 changes: 46 additions & 17 deletions lib/index.js
View file
Expand Up @@ -105,6 +105,12 @@ class Parser extends Transform {
}else{
options.escape = options.escape[0]
}
// Normalize option `info`
if(options.info === undefined || options.info === null || options.info === false){
options.info = false
}else if(options.info !== true){
throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`)
}
// Normalize option `quote`
if(options.quote === null || options.quote === false || options.quote === ''){
options.quote = null
Expand Down Expand Up @@ -151,6 +157,13 @@ class Parser extends Transform {
}else if(options.rtrim !== true){
options.rtrim = false
}
this.info = {
comment_lines: 0,
empty_lines: 0,
invalid_field_length: 0,
lines: 1,
records: 0
}
this.options = options
this.state = {
castField: fnCastField,
Expand All @@ -161,6 +174,7 @@ class Parser extends Transform {
expectedRecordLength: options.columns === null ? 0 : options.columns.length,
field: new ResizeableBuffer(20),
firstLineToHeaders: fnFirstLineToHeaders,
info: Object.assign({}, this.info),
previousBuf: undefined,
quoting: false,
stop: false,
Expand All @@ -173,13 +187,6 @@ class Parser extends Transform {
wasQuoting: false,
wasRowDelimiter: false
}
this.info = {
comment_lines: 0,
empty_lines: 0,
invalid_field_length: 0,
lines: 1,
records: 0
}
}
_transform(buf, encoding, callback){
if(this.state.stop === true){
Expand All @@ -199,7 +206,7 @@ class Parser extends Transform {
callback(err)
}
__parse(nextBuf, end){
const {comment, escape, from, from_line, ltrim, max_record_size, quote, raw, relax, rtrim, skip_empty_lines, to, to_line} = this.options
const {comment, escape, from, from_line, info, ltrim, max_record_size, quote, raw, relax, rtrim, skip_empty_lines, to, to_line} = this.options
let {record_delimiter} = this.options
const {previousBuf, rawBuffer, escapeIsQuote, trimChars} = this.state
let buf
Expand All @@ -221,6 +228,10 @@ class Parser extends Transform {
}
if(this.state.wasRowDelimiter === true){
this.info.lines++
console.log(this.info.records, this.state.info.records)
if(info === true && this.state.record.length === 0 && this.state.field.length === 0 && this.state.wasQuoting === false){
this.state.info = Object.assign({}, this.info)
}
this.state.wasRowDelimiter = false
}
if(to_line !== -1 && this.info.lines > to_line){
Expand Down Expand Up @@ -356,12 +367,13 @@ class Parser extends Transform {
}
}
}
if(this.state.commenting === false && max_record_size !== 0){
if(this.state.record_length + this.state.field.length > max_record_size){
if(this.state.commenting === false){
if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){
const err = this.__error(`Max Record Size: record exceed the maximum number of tolerated bytes of ${max_record_size} on line ${this.info.lines}`)
if(err !== undefined) return err
}
}

const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr)
// rtrim in non quoting is handle in __onField
const rappend = rtrim === false || this.state.wasQuoting === false
Expand Down Expand Up @@ -399,7 +411,7 @@ class Parser extends Transform {
return chr === space || chr === cr || chr === nl
}
__onRow(){
const {columns, from, relax_column_count, raw, skip_lines_with_empty_values} = this.options
const {columns, info, from, relax_column_count, raw, skip_lines_with_empty_values} = this.options
const {enabled, record} = this.state
// Validate column length
if(columns === true && this.state.firstLineToHeaders){
Expand Down Expand Up @@ -441,27 +453,40 @@ class Parser extends Transform {
if(from === 1 || this.info.records >= from){
if(columns !== false){
const obj = {}
// Transform record array to an object
for(let i in record){
if(columns[i].disabled) continue
obj[columns[i].name] = record[i]
}
const {objname} = this.options
if(objname === undefined){
if(raw === true){
this.push({raw: this.state.rawBuffer.toString(), record: obj})
if(raw === true || info === true){
this.push(Object.assign(
{record: obj},
(raw === true ? {raw: this.state.rawBuffer.toString()}: {}),
(info === true ? {info: this.state.info}: {})
))
}else{
this.push(obj)
}
}else{
if(raw === true){
this.push({raw: this.state.rawBuffer.toString(), record: [obj[objname], obj]})
if(raw === true || info === true){
this.push(Object.assign(
{record: [obj[objname], obj]},
raw === true ? {raw: this.state.rawBuffer.toString()}: {},
info === true ? {info: this.state.info}: {}
))
}else{
this.push([obj[objname], obj])
}
}
}else{
if(raw === true){
this.push({raw: this.state.rawBuffer.toString(), record: record})
if(raw === true || info === true){
this.push(Object.assign(
{record: record},
raw === true ? {raw: this.state.rawBuffer.toString()}: {},
info === true ? {info: this.state.info}: {}
))
}else{
this.push(record)
}
Expand All @@ -485,11 +510,15 @@ class Parser extends Transform {
}
}
__resetRow(){
const {info} = this.options
if(this.options.raw === true){
this.state.rawBuffer.reset()
}
this.state.record = []
this.state.record_length = 0
// if(info === true){
// this.state.info = null
// }
}
__onField(){
const {cast, rtrim} = this.options
Expand Down
7 changes: 6 additions & 1 deletion test/api.types.ts
View file
Expand Up @@ -12,7 +12,7 @@ describe('API Types', () => {
const keys: any = Object.keys(options)
keys.sort().should.eql([
'cast', 'cast_date', 'columns', 'comment', 'delimiter',
'escape', 'from', 'from_line', 'ltrim', 'max_record_size',
'escape', 'from', 'from_line', 'info', 'ltrim', 'max_record_size',
'objname', 'quote', 'raw', 'readableObjectMode',
'record_delimiter', 'relax', 'relax_column_count',
'rtrim', 'skip_empty_lines', 'skip_lines_with_empty_values',
Expand Down Expand Up @@ -127,6 +127,11 @@ describe('API Types', () => {
options.from_line = 10
})

it('info', () => {
const options: Options = {}
options.info = true
})

it('ltrim', () => {
const options: Options = {}
options.ltrim = true
Expand Down

0 comments on commit 8c8132a

Please sign in to comment.