Added tab to bad characters, increased batch size

meekyphotos · Jul 18, 2021 · cb42f3b · cb42f3b
1 parent 896efe1
commit cb42f3b
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 4 deletions.
diff --git a/core/commands/connectors/db_connector.go b/core/commands/connectors/db_connector.go
@@ -74,8 +74,7 @@ func (p *PgConnector) Write(data []map[string]interface{}) error {
 				value := row[c.Name]
 				switch value.(type) {
 				case string: // already marshalled
-					repairedString := string(carriageReturn.ReplaceAll([]byte(value.(string)), []byte{}))
-					vals[i] = repairedString
+					vals[i] = value
 				default:
 					marshal, err := json.Marshal(value)
 					if err != nil {

diff --git a/core/commands/load_osm.go b/core/commands/load_osm.go
@@ -52,7 +52,7 @@ func (r OsmRunner) Run(c *utils.Config) error {
 	if err != nil {
 		return err
 	}
-	requests := pipeline.BatchRequest(channel, 2500, time.Second)
+	requests := pipeline.BatchRequest(channel, 10000, time.Second)
 	var pgWorkers sync.WaitGroup
 	pgWorkers.Add(1)
 	beat := &pipeline.ProgressBarBeat{OperationName: "Writing"}

diff --git a/core/commands/osm/decode_tag.go b/core/commands/osm/decode_tag.go
@@ -91,7 +91,7 @@ var keyVal = []byte(`":"`)
 var quotes = []byte(`"`)
 var endPar = []byte(`}`)
 var nameBytes = []byte(`name`)
-var carriageReturn = regexp.MustCompile(`[\n\r"\\]`)
+var carriageReturn = regexp.MustCompile(`[\n\r\t"\\]`)
 var escapeQuote = regexp.MustCompile(`"`)
 
 // Make tags map from stringtable and array of IDs (used in DenseNodes encoding).