# HG changeset patch # User Sascha L. Teichmann # Date 1540305806 -7200 # Node ID 0522289e85031921392d70a3227428ae8a209b00 # Parent d2f30a784fb3bebe16fae3ce0f53a8155b817a49 Removed the tools soundingresults, tin2octree and octree2contour. They are now all part of the importer job for sounding results. diff -r d2f30a784fb3 -r 0522289e8503 cmd/octree2contour/db.go --- a/cmd/octree2contour/db.go Tue Oct 23 14:28:43 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,39 +0,0 @@ -package main - -import ( - "database/sql" - "flag" - - "github.com/jackc/pgx" - "github.com/jackc/pgx/stdlib" -) - -var ( - dbhost = flag.String("dbhost", "localhost", "database host") - dbport = flag.Uint("dbport", 5432, "database port") - dbname = flag.String("dbname", "gemma", "database user") - dbuser = flag.String("dbuser", "scott", "database user") - dbpassword = flag.String("dbpw", "tiger", "database password") - dbssl = flag.String("dbssl", "prefer", "database SSL mode") -) - -func run(fn func(*sql.DB) error) error { - - // To ease SSL config ride a bit on parsing. - cc, err := pgx.ParseConnectionString("sslmode=" + *dbssl) - if err != nil { - return err - } - - // Do the rest manually to allow whitespace in user/password. - cc.Host = *dbhost - cc.Port = uint16(*dbport) - cc.User = *dbuser - cc.Password = *dbpassword - cc.Database = *dbname - - db := stdlib.OpenDB(cc) - defer db.Close() - - return fn(db) -} diff -r d2f30a784fb3 -r 0522289e8503 cmd/octree2contour/main.go --- a/cmd/octree2contour/main.go Tue Oct 23 14:28:43 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,122 +0,0 @@ -package main - -import ( - "flag" - "log" - "runtime" - "sort" - "sync" - "time" - - "gemma.intevation.de/gemma/pkg/octree" -) - -var ( - one = flag.Bool("o", false, "create only a single contour") - step = flag.Float64("s", 0.5, "step width") - tol = flag.Float64("t", 0.1, "tolerance for simplification") - max = flag.Float64("m", 10, "max height from lowest point") - bottleneck = flag.String("bottleneck", "", "bottleneck id") - date = flag.String("date", "", "date info") -) - -func processLevels( - tree *octree.Tree, - jobs <-chan float64, - results chan<- result, - wg *sync.WaitGroup, -) { - defer wg.Done() - for h := range jobs { - var lines octree.MultiLineStringZ - tree.Horizontal(h, func(t *octree.Triangle) { - line := t.IntersectHorizontal(h) - if len(line) > 1 { - lines = append(lines, line) - } - }) - lines = lines.Merge() - results <- result{h, lines} - } -} - -func process(tree *octree.Tree) []result { - - if *one { - var lines octree.MultiLineStringZ - tree.Horizontal(*step, func(t *octree.Triangle) { - line := t.IntersectHorizontal(*step) - if len(line) > 0 { - lines = append(lines, line) - } - }) - lines = lines.Merge() - return []result{{*step, lines}} - } - - results := make(chan result) - done := make(chan struct{}) - - var all []result - go func() { - for { - select { - case r := <-results: - all = append(all, r) - case <-done: - return - } - } - }() - - var wg sync.WaitGroup - jobs := make(chan float64) - for i, n := 0, runtime.NumCPU(); i < n; i++ { - wg.Add(1) - go processLevels(tree, jobs, results, &wg) - } - for h := tree.Min.Z; h <= tree.Max.Z; h += *step { - jobs <- h - } - close(jobs) - wg.Wait() - done <- struct{}{} - - sort.Slice(all, func(i, j int) bool { - return all[i].h < all[j].h - }) - - return all -} - -func main() { - flag.Parse() - - if *bottleneck == "" || *date == "" { - log.Fatalln("missing bottleneck or date option.") - } - - dateInfo, err := time.Parse("2006-01-02", *date) - if err != nil { - log.Fatalf("error: %v\n", err) - } - - for _, fname := range flag.Args() { - log.Printf("processing %s\n", fname) - start := time.Now() - tree, err := octree.LoadTree(fname) - if err != nil { - log.Printf("error: %v\n", err) - continue - } - log.Printf("loading took: %v\n", time.Since(start)) - start = time.Now() - all := process(tree) - log.Printf("processing took: %v\n", time.Since(start)) - start = time.Now() - if err = store(all, tree.EPSG, *bottleneck, dateInfo, *tol); err != nil { - log.Printf("error: %v\n", err) - } - log.Printf("storing took: %v\n", time.Since(start)) - } -} diff -r d2f30a784fb3 -r 0522289e8503 cmd/octree2contour/store.go --- a/cmd/octree2contour/store.go Tue Oct 23 14:28:43 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,83 +0,0 @@ -package main - -import ( - "database/sql" - "time" - - "gemma.intevation.de/gemma/pkg/octree" -) - -type result struct { - h float64 - lines octree.MultiLineStringZ -} - -const ( - deleteSQL = ` -DELETE FROM waterway.sounding_results_contour_lines -WHERE sounding_result_id IN (SELECT id -FROM waterway.sounding_results -WHERE bottleneck_id = $1 AND date_info = $2) -` - insertSQL = ` -INSERT INTO waterway.sounding_results_contour_lines - (sounding_result_id, height, lines) -SELECT - sr.id, - $1, - ST_Transform( - ST_Multi( - ST_CollectionExtract( - ST_Intersection( - ST_Transform(sr.area::geometry, $3::integer), - ST_SimplifyPreserveTopology( - ST_LineMerge(ST_GeomFromWKB($2, $3::integer)), - $6 - ) - ), - 2 - ) - ), - 4326 - ) -FROM waterway.sounding_results sr -WHERE bottleneck_id = $4 AND date_info = $5 -` -) - -func store( - all []result, epsg uint32, - bottleneck string, date time.Time, - tol float64, -) error { - - return run(func(db *sql.DB) error { - - tx, err := db.Begin() - if err != nil { - return err - } - defer tx.Rollback() - - if _, err := tx.Exec(deleteSQL, bottleneck, date); err != nil { - return err - } - - stmt, err := tx.Prepare(insertSQL) - if err != nil { - return err - } - - for _, r := range all { - if _, err := stmt.Exec( - r.h, r.lines.AsWKB2D(), epsg, - bottleneck, date, - tol, - ); err != nil { - return err - } - } - - return tx.Commit() - }) -} diff -r d2f30a784fb3 -r 0522289e8503 cmd/soundingresults/main.go --- a/cmd/soundingresults/main.go Tue Oct 23 14:28:43 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,286 +0,0 @@ -package main - -import ( - "bufio" - "database/sql" - "encoding/hex" - "flag" - "fmt" - "io" - "log" - "os" - "path/filepath" - "runtime" - "sort" - "strings" - "sync" - "time" - - "github.com/jackc/pgx" - "github.com/jackc/pgx/stdlib" -) - -var ( - dump = flag.Bool("dump", true, "dump SQL insert statements") - insecure = flag.Bool("insecure", false, "skip SSL verification") - dbhost = flag.String("dbhost", "localhost", "database host") - dbport = flag.Uint("dbport", 5432, "database port") - dbname = flag.String("dbname", "gemma", "database user") - dbuser = flag.String("dbuser", "scott", "database user") - dbpassword = flag.String("dbpw", "tiger", "database password") - dbssl = flag.String("dbssl", "prefer", "database SSL mode") -) - -func run(fn func(*sql.DB) error) error { - - // To ease SSL config ride a bit on parsing. - cc, err := pgx.ParseConnectionString("sslmode=" + *dbssl) - if err != nil { - return err - } - - // Do the rest manually to allow whitespace in user/password. - cc.Host = *dbhost - cc.Port = uint16(*dbport) - cc.User = *dbuser - cc.Password = *dbpassword - cc.Database = *dbname - - db := stdlib.OpenDB(cc) - defer db.Close() - - return fn(db) -} - -// TODO: This should come from the depth_references table. -var knownDepthReferences = []string{ - "ADR", "ETRS", "FZP", "GLW", "HBO", - "HDC", "HNW", "HSW", "IGN", "KP", - "LDC", "LNW", "NAP", "NGM", "POT", - "PUL", "RN", "TAW", "WGS", "ZPG", -} - -func isKnownDepthReference(ref string) bool { - ref = strings.ToUpper(ref) - for _, r := range knownDepthReferences { - if r == ref { - return true - } - } - return false -} - -type meta struct { - date time.Time - name string - depthReference string -} - -func substituteName(fname, name string) string { - dir := filepath.Dir(fname) - info := filepath.Join(dir, "INFO.txt") - f, err := os.Open(info) - if err != nil { - log.Printf("warn: %v\n", err) - return name - } - defer f.Close() - - s := bufio.NewScanner(f) - - for search := strings.ToLower(name); s.Scan(); { - line := strings.TrimSpace(s.Text()) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - if parts := strings.SplitN(line, "=", 2); len(parts) == 2 && - strings.TrimSpace(strings.ToLower(parts[0])) == search { - return strings.TrimSpace(parts[1]) - } - } - - if err := s.Err(); err != nil { - log.Printf("error: %v\n", err) - } - - return name -} - -func parseFilename(fname string) (*meta, error) { - - base := filepath.Base(fname) - - compressed := strings.ToLower(filepath.Ext(base)) - for _, ext := range []string{".gz", ".bz2"} { - if ext == compressed { - base = base[:len(base)-len(ext)] - break - } - } - - // Cut .txt - base = base[:len(base)-len(filepath.Ext(base))] - - if !strings.HasSuffix(strings.ToUpper(base), "_WGS84") { - return nil, fmt.Errorf("%s is not in WGS84", base) - } - - base = base[:len(base)-len("_WGS84")] - - idx := strings.IndexRune(base, '_') - if idx == -1 { - return nil, fmt.Errorf("%s has no date", base) - } - - datePart := base[:idx] - - date, err := time.Parse("20060102", datePart) - if err != nil { - return nil, fmt.Errorf("error %s: %v\n", datePart, err) - } - - rest := base[idx+1:] - - if idx = strings.LastIndex(rest, "_"); idx == -1 { - return nil, fmt.Errorf("%s has no depth reference", base) - } - - depthReference := rest[idx+1:] - - if !isKnownDepthReference(depthReference) { - return nil, fmt.Errorf( - "%s is not a known depth reference", depthReference) - } - - rest = rest[:idx] - - if !strings.HasSuffix(strings.ToUpper(rest), "_MB") { - return nil, fmt.Errorf("%s is not in WGS84", base) - } - - name := rest[:len(rest)-len("_MB")] - - name = substituteName(fname, name) - - return &meta{ - name: name, - depthReference: depthReference, - date: date, - }, nil -} - -type result struct { - m *meta - points points3d - wkb string -} - -func processor(fnames <-chan string, results chan<- result, wg *sync.WaitGroup) { - defer wg.Done() - - for fname := range fnames { - log.Printf("Processing %s\n", fname) - m, err := parseFilename(fname) - if err != nil { - log.Printf("error: %v\n", err) - continue - } - _ = m - points, err := parseXYZ(fname) - if err != nil { - log.Printf("error: %v\n", err) - continue - } - log.Printf("Number of points: %d\n", len(points)) - - wkb := points.asWKB() - log.Printf("WKB size %.2f MB\n", float64(len(wkb))/(1024*1024)) - - results <- result{m, points, wkb} - } -} - -func quote(s string) string { - return "'" + strings.Replace(s, "'", "'''", -1) + "'" -} - -func main() { - flag.Parse() - - var wg sync.WaitGroup - - fnames := make(chan string) - results := make(chan result) - done := make(chan struct{}) - - handler := func(result) {} - flush := func() {} - - if *dump { - var results []result - handler = func(r result) { results = append(results, r) } - flush = func() { - sort.Slice(results, func(i, j int) bool { - if a, b := results[i].m.name, results[j].m.name; a != b { - return a < b - } - return results[i].m.date.Before(results[j].m.date) - }) - out := bufio.NewWriter(os.Stdout) - fmt.Fprintln(out, "BEGIN;") - for i := range results { - r := &results[i] - fmt.Fprintln(out, "INSERT INTO waterway.sounding_results (") - fmt.Fprintln(out, " bottleneck_id,") - fmt.Fprintln(out, " date_info,") - fmt.Fprintln(out, " depth_reference,") - fmt.Fprintln(out, " point_cloud") - fmt.Fprintln(out, ") VALUES (") - fmt.Fprintf(out, - " (SELECT bottleneck_id from waterway.bottlenecks where objnam = %s),\n", - quote(r.m.name)) - fmt.Fprintf(out, " '%s'::date,\n", r.m.date.Format("2006-01-02")) - fmt.Fprintf(out, " %s,\n", quote(r.m.depthReference)) - fmt.Fprintf(out, " '") - io.Copy(hex.NewEncoder(out), strings.NewReader(r.wkb)) - fmt.Fprintln(out, "'") - fmt.Fprintln(out, ");") - } - fmt.Fprintln(out, "COMMIT;") - out.Flush() - } - } else { - // TODO: Implement database stuff. - } - - fin := make(chan struct{}) - - go func() { - defer func() { flush(); close(fin) }() - for { - select { - case <-done: - return - case r := <-results: - handler(r) - } - } - }() - - for i, n := 0, runtime.NumCPU(); i < n; i++ { - wg.Add(1) - go processor(fnames, results, &wg) - } - - for _, fname := range flag.Args() { - fnames <- fname - } - - close(fnames) - - wg.Wait() - - close(done) - <-fin -} diff -r d2f30a784fb3 -r 0522289e8503 cmd/soundingresults/points.go --- a/cmd/soundingresults/points.go Tue Oct 23 14:28:43 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,129 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "compress/bzip2" - "compress/gzip" - "encoding/binary" - "io" - "log" - "math" - "os" - "path/filepath" - "strconv" - "strings" -) - -type point3d struct { - x float64 - y float64 - z float64 -} - -type points3d []*point3d - -func parseXYZ(fname string) (points3d, error) { - f, err := os.Open(fname) - if err != nil { - return nil, err - } - defer f.Close() - - r, err := wrap(fname, f) - if err != nil { - return nil, err - } - return readXYZ(r) -} - -func wrap(fname string, f io.Reader) (io.Reader, error) { - - switch strings.ToLower(filepath.Ext(fname)) { - case ".gz": - return gzip.NewReader(f) - case ".bz2": - return bzip2.NewReader(f), nil - } - - return bufio.NewReader(f), nil -} - -func readXYZ(r io.Reader) (points3d, error) { - - // Alloc in larger chunks to reduce pressure on memory management. - var chunk []point3d - alloc := func() *point3d { - if len(chunk) == 0 { - chunk = make([]point3d, 8*1024) - } - p := &chunk[0] - chunk = chunk[1:] - return p - } - - var points points3d - - s := bufio.NewScanner(r) - if s.Scan() { // Skip header line. - for line := 2; s.Scan(); line++ { - p := alloc() - text := s.Text() - // fmt.Sscanf(text, "%f,%f,%f") is 4 times slower. - idx := strings.IndexByte(text, ',') - if idx == -1 { - log.Printf("format error in line %d\n", line) - continue - } - var err error - if p.x, err = strconv.ParseFloat(text[:idx], 64); err != nil { - log.Printf("format error in line %d: %v\n", line, err) - continue - } - text = text[idx+1:] - if idx = strings.IndexByte(text, ','); idx == -1 { - log.Printf("format error in line %d\n", line) - continue - } - if p.y, err = strconv.ParseFloat(text[:idx], 64); err != nil { - log.Printf("format error in line %d: %v\n", line, err) - continue - } - text = text[idx+1:] - if p.z, err = strconv.ParseFloat(text, 64); err != nil { - log.Printf("format error in line %d: %v\n", line, err) - continue - } - points = append(points, p) - } - } - - return points, s.Err() -} - -const ( - wkbNDR byte = 1 - wkbPointZ uint32 = 1 + 1000 - wkbMultiPointZ uint32 = 4 + 1000 -) - -func (ps points3d) asWKB() string { - - size := 1 + 4 + 4 + len(ps)*(1+4+3*8) - - buf := bytes.NewBuffer(make([]byte, 0, size)) - - binary.Write(buf, binary.LittleEndian, wkbNDR) - binary.Write(buf, binary.LittleEndian, wkbMultiPointZ) - binary.Write(buf, binary.LittleEndian, uint32(len(ps))) - - for _, p := range ps { - binary.Write(buf, binary.LittleEndian, wkbNDR) - binary.Write(buf, binary.LittleEndian, wkbPointZ) - binary.Write(buf, binary.LittleEndian, math.Float64bits(p.x)) - binary.Write(buf, binary.LittleEndian, math.Float64bits(p.y)) - binary.Write(buf, binary.LittleEndian, math.Float64bits(p.z)) - } - - return buf.String() -} diff -r d2f30a784fb3 -r 0522289e8503 cmd/tin2octree/main.go --- a/cmd/tin2octree/main.go Tue Oct 23 14:28:43 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,184 +0,0 @@ -package main - -import ( - "bytes" - "context" - "crypto/sha1" - "database/sql" - "encoding/base64" - "flag" - "fmt" - "io" - "log" - "math" - "os" - "time" - - "gemma.intevation.de/gemma/pkg/octree" - "github.com/jackc/pgx" - "github.com/jackc/pgx/stdlib" -) - -var ( - bottleneck = flag.String("bottleneck", "", "bottleneck id") - date = flag.String("date", "", "date info") - file = flag.String("file", "", "save to file") - insert = flag.Bool("insert", false, "write as SQL insert statement") - - dbhost = flag.String("dbhost", "localhost", "database host") - dbport = flag.Uint("dbport", 5432, "database port") - dbname = flag.String("dbname", "gemma", "database user") - dbuser = flag.String("dbuser", "scott", "database user") - dbpassword = flag.String("dbpw", "tiger", "database password") - dbssl = flag.String("dbssl", "prefer", "database SSL mode") -) - -func run(fn func(*sql.DB) error) error { - - // To ease SSL config ride a bit on parsing. - cc, err := pgx.ParseConnectionString("sslmode=" + *dbssl) - if err != nil { - return err - } - - // Do the rest manually to allow whitespace in user/password. - cc.Host = *dbhost - cc.Port = uint16(*dbport) - cc.User = *dbuser - cc.Password = *dbpassword - cc.Database = *dbname - - db := stdlib.OpenDB(cc) - defer db.Close() - - return fn(db) -} - -const ( - centroidSQL = ` -SELECT ST_X(ST_Centroid(point_cloud::geometry)), ST_Y(ST_Centroid(point_cloud::geometry)) -FROM waterway.sounding_results -WHERE bottleneck_id = $1 AND date_info = $2 -` -) - -func utmZone(x, y float64) uint32 { - var pref uint32 - if y > 0 { - pref = 32600 - } else { - pref = 32700 - } - zone := uint32(math.Floor((x+180)/6)) + 1 - return zone + pref -} - -func main() { - flag.Parse() - - if *bottleneck == "" || *date == "" { - log.Fatalln("missing bottleneck or date option.") - } - - dateInfo, err := time.Parse("2006-01-02", *date) - if err != nil { - log.Fatalf("error: %v\n", err) - } - - var t *octree.Tin - - ctx := context.Background() - - if err := run(func(db *sql.DB) error { - conn, err := db.Conn(ctx) - if err != nil { - return err - } - defer conn.Close() - - var utmZ uint32 - var cx, cy float64 - - conn.QueryRowContext(ctx, centroidSQL, *bottleneck, dateInfo).Scan(&cx, &cy) - switch { - case err == sql.ErrNoRows: - return nil - case err != nil: - return err - } - log.Printf("lat/lon: [%f, %f]\n", cx, cy) - utmZ = utmZone(cx, cy) - log.Printf("UTM zone: %d\n", utmZ) - - start := time.Now() - t, err = octree.GenerateTin(conn, ctx, *bottleneck, dateInfo, utmZ) - log.Printf("query took: %s\n", time.Since(start)) - return err - }); err != nil { - log.Fatalf("error: %v\n", err) - } - - if t == nil { - log.Fatalf("error: No such sounding result (%s, %s)\n", - *bottleneck, dateInfo) - } - - tb := octree.NewBuilder(t) - tb.Build() - - if *insert { - var w io.Writer - var f *os.File - - if *file != "" { - if f, err = os.Create(*file); err != nil { - log.Fatalf("error: %v\n", err) - } - w = f - } else { - w = os.Stdout - } - - var buf bytes.Buffer - if err := tb.WriteTo(&buf); err != nil { - log.Fatalf("error: %v\n", err) - } - data := buf.String() - h := sha1.New() - buf.WriteTo(h) - fmt.Fprintln(w, "BEGIN;") - fmt.Fprintln(w, "INSERT INTO waterway.octrees") - fmt.Fprintf(w, "SELECT sr.id, '%x',\n", h.Sum(nil)) - fmt.Fprint(w, "decode('") - fmt.Fprintf(w, "%s", base64.StdEncoding.EncodeToString([]byte(data))) - fmt.Fprintln(w, "', 'base64')") - fmt.Fprintln(w, "FROM waterway.sounding_results sr") - fmt.Fprintf(w, - "WHERE sr.bottleneck_id = '%s' AND sr.date_info = '%s'::date;\n", - *bottleneck, *date) - fmt.Fprintln(w, "END;") - - if f != nil { - if err := f.Close(); err != nil { - log.Fatalf("error: %v\n", err) - } - } - return - } - - if *file != "" { - f, err := os.Create(*file) - if err != nil { - log.Printf("error: %v\n", err) - } - err = tb.WriteTo(f) - if err2 := f.Close(); err == nil { - if err != nil { - err = err2 - } - } - if err != nil { - log.Fatalf("error: %v\n", err) - } - } -}