Mercurial > gemma
changeset 4653:8efc6b3289f3 stree-experiment
Merged default into stree-experiment branch.
| author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
|---|---|
| date | Mon, 14 Oct 2019 12:26:00 +0200 |
| parents | f5492fda097c (diff) e9a99e81f723 (current diff) |
| children | 3eda5a7215ab |
| files | |
| diffstat | 10 files changed, 453 insertions(+), 22 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cmd/oct2str/main.go Mon Oct 14 12:26:00 2019 +0200 @@ -0,0 +1,167 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2019 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Sascha L. Teichmann <sascha.teichmann@intevation.de> + +package main + +import ( + "context" + "crypto/sha1" + "database/sql" + "encoding/hex" + "flag" + "log" + + "gemma.intevation.de/gemma/pkg/octree" + "github.com/jackc/pgx" + "github.com/jackc/pgx/stdlib" +) + +const ( + fetchOneOctreeSQL = ` +SELECT + id, + octree_index +FROM waterway.sounding_results +WHERE mesh_index IS NULL +LIMIT 1` + + storeSTRTreeSQL = ` +UPDATE waterway.sounding_results +SET mesh_index = $1, mesh_checksum = $2 +WHERE id = $3` +) + +func process(ctx context.Context, conn *sql.Conn, count int) error { + + fetch, err := conn.PrepareContext(ctx, fetchOneOctreeSQL) + if err != nil { + return err + } + defer fetch.Close() + + store, err := conn.PrepareContext(ctx, storeSTRTreeSQL) + if err != nil { + return err + } + defer store.Close() + + var next func() bool + if count < 0 { + next = func() bool { return true } + } else { + var c int + next = func() bool { + if c < count { + c++ + return true + } + return false + } + } + +loop: + for next() { + switch err := func() error { + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + var id int64 + var data []byte + + if err = tx.Stmt(fetch).QueryRowContext(ctx).Scan(&id, &data); err != nil { + return err + } + + otree, err := octree.Deserialize(data) + if err != nil { + return err + } + + unused := otree.FindUnused() + + log.Printf("unused: %d\n", len(unused)) + + str := octree.STRTree{Entries: 16} + str.BuildWithout(otree.Tin(), unused) + + out, err := str.Bytes() + if err != nil { + return err + } + h := sha1.New() + h.Write(out) + checksum := hex.EncodeToString(h.Sum(nil)) + log.Printf("hash: %s\n", checksum) + + if _, err := tx.Stmt(store).ExecContext(ctx, out, checksum, id); err != nil { + return err + } + + return tx.Commit() + }(); { + case err == sql.ErrNoRows: + break loop + case err != nil: + return err + } + } + + return nil +} + +func connect(cc pgx.ConnConfig, count int) error { + db := stdlib.OpenDB(cc) + defer db.Close() + + ctx := context.Background() + + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + + return process(ctx, conn, count) +} + +func main() { + var ( + db = flag.String("database", "gemma", "database name") + user = flag.String("user", "sophie", "database user") + host = flag.String("host", "localhost", "database host") + password = flag.String("password", "so2Phie4", "database password") + port = flag.Uint("port", 5432, "database port") + ssl = flag.String("ssl", "prefer", "SSL mode") + count = flag.Int("count", -1, "how many sounding results to convert") + ) + + flag.Parse() + cc, err := pgx.ParseConnectionString("sslmode=" + *ssl) + if err != nil { + log.Fatalf("error: %v\n", err) + } + + // Do the rest manually to allow whitespace in user/password. + cc.Host = *host + cc.Port = uint16(*port) + cc.User = *user + cc.Password = *password + cc.Database = *db + + if err := connect(cc, *count); err != nil { + log.Fatalf("error: %v\n", err) + } +}
--- a/go.mod Mon Oct 14 08:25:13 2019 +0200 +++ b/go.mod Mon Oct 14 12:26:00 2019 +0200 @@ -6,6 +6,7 @@ github.com/cockroachdb/apd v1.1.0 // indirect github.com/etcd-io/bbolt v1.3.3 github.com/fogleman/contourmap v0.0.0-20190814184649-9f61d36c4199 + github.com/frankban/quicktest v1.5.0 // indirect github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/golang/snappy v0.0.1 github.com/gorilla/mux v1.7.3 @@ -16,6 +17,7 @@ github.com/magiconair/properties v1.8.1 // indirect github.com/mitchellh/go-homedir v1.1.0 github.com/pelletier/go-toml v1.4.0 // indirect + github.com/pierrec/lz4 v2.3.0+incompatible github.com/pkg/errors v0.8.1 // indirect github.com/rs/cors v1.7.0 github.com/sergi/go-diff v1.0.0
--- a/go.sum Mon Oct 14 08:25:13 2019 +0200 +++ b/go.sum Mon Oct 14 12:26:00 2019 +0200 @@ -29,6 +29,8 @@ github.com/fogleman/contourmap v0.0.0-20190814184649-9f61d36c4199 h1:kufr0u0RIG5ACpjFsPRbbuHa0FhMWsS3tnSFZ2hf07s= github.com/fogleman/contourmap v0.0.0-20190814184649-9f61d36c4199/go.mod h1:mqaaaP4j7nTF8T/hx5OCljA7BYWHmrH2uh+Q023OchE= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/frankban/quicktest v1.5.0 h1:Tb4jWdSpdjKzTUicPnY61PZxKbDoGa7ABbrReT3gQVY= +github.com/frankban/quicktest v1.5.0/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -50,6 +52,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -95,6 +99,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pierrec/lz4 v2.3.0+incompatible h1:CZzRn4Ut9GbUkHlQ7jqBXeZQV41ZSKWFc302ZU6lUTk= +github.com/pierrec/lz4 v2.3.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
--- a/pkg/imports/sr.go Mon Oct 14 08:25:13 2019 +0200 +++ b/pkg/imports/sr.go Mon Oct 14 12:26:00 2019 +0200 @@ -24,6 +24,7 @@ "errors" "fmt" "io" + "log" "math" "os" "path" @@ -565,6 +566,16 @@ feedback.Info("Clipping STR tree took %v.", time.Since(start)) feedback.Info("Number of triangles to clip %d.", len(removed)) + start = time.Now() + s := octree.STRTree{Entries: 16} + s.BuildWithout(tin, removed) + if _, err2 := s.Bytes(); err2 != nil { + log.Printf("serializing STRTree failed: %v\n", err2) + } + log.Printf("Building strtree took: %v.\n", time.Since(start)) + + // return nil, UnchangedError("nothing to do") + feedback.Info("Build final octree index") builder := octree.NewBuilder(tin)
--- a/pkg/octree/loader.go Mon Oct 14 08:25:13 2019 +0200 +++ b/pkg/octree/loader.go Mon Oct 14 12:26:00 2019 +0200 @@ -20,55 +20,130 @@ "log" "github.com/golang/snappy" + "github.com/pierrec/lz4" ) -func loadReader(r *bufio.Reader) (*Tree, error) { - tree := new(Tree) +func (s *STRTree) deserializeIndex(r *bufio.Reader) error { + var numIndex int32 + if err := binary.Read(r, binary.LittleEndian, &numIndex); err != nil { + return err + } + index := make([]int32, numIndex) + + var last int32 + for i := range index { + v, err := binary.ReadVarint(r) + if err != nil { + return err + } + value := int32(v) + last + index[i] = value + last = value + } - if err := binary.Read(r, binary.LittleEndian, &tree.EPSG); err != nil { - return nil, err + return nil +} + +func (s *STRTree) deserializeBBoxes(r *bufio.Reader) error { + + var numBBoxes int32 + if err := binary.Read(r, binary.LittleEndian, &numBBoxes); err != nil { + return err + } + + bboxes := make([]Box2D, numBBoxes) + s.bboxes = bboxes + + var err error + + read := func(v *float64) { + if err == nil { + err = binary.Read(r, binary.LittleEndian, v) + } } - log.Printf("info: EPSG: %d\n", tree.EPSG) - - if err := tree.Min.Read(r); err != nil { - return nil, err + for i := range bboxes { + read(&bboxes[i].X1) + read(&bboxes[i].Y1) + read(&bboxes[i].X2) + read(&bboxes[i].Y2) } - if err := tree.Max.Read(r); err != nil { - return nil, err + return err +} + +func (s *STRTree) deserialize(r *bufio.Reader) error { + s.tin = new(Tin) + + if err := s.tin.Deserialize(r); err != nil { + return err + } + var numEntries uint8 + if err := binary.Read(r, binary.LittleEndian, &numEntries); err != nil { + return err + } + s.Entries = int(numEntries) + + if err := s.deserializeIndex(r); err != nil { + return err + } + + return s.deserializeBBoxes(r) +} + +func (s *STRTree) FromBytes(data []byte) error { + return s.deserialize( + bufio.NewReader( + lz4.NewReader( + bytes.NewReader(data)))) +} + +func (t *Tin) Deserialize(r *bufio.Reader) error { + + if err := binary.Read(r, binary.LittleEndian, &t.EPSG); err != nil { + return err + } + + log.Printf("info: EPSG: %d\n", t.EPSG) + + if err := t.Min.Read(r); err != nil { + return err + } + + if err := t.Max.Read(r); err != nil { + return err } log.Printf("info: BBOX: [[%f, %f, %f], [%f, %f, %f]]\n", - tree.Min.X, tree.Min.Y, tree.Min.Z, - tree.Max.X, tree.Max.Y, tree.Max.Z) + t.Min.X, t.Min.Y, t.Min.Z, + t.Max.X, t.Max.Y, t.Max.Z) var numVertices uint32 if err := binary.Read(r, binary.LittleEndian, &numVertices); err != nil { - return nil, err + return err } log.Printf("info: vertices: %d\n", numVertices) vertices := make([]Vertex, numVertices) - tree.vertices = vertices + t.Vertices = vertices for i := range vertices { if err := vertices[i].Read(r); err != nil { - return nil, err + return err } } var numTriangles uint32 if err := binary.Read(r, binary.LittleEndian, &numTriangles); err != nil { - return nil, err + return err } log.Printf("info: triangles: %d\n", numTriangles) indices := make([]int32, 3*numTriangles) triangles := make([][]int32, numTriangles) - tree.triangles = triangles + t.Triangles = triangles var last int32 @@ -79,7 +154,7 @@ for j := range tri { v, err := binary.ReadVarint(r) if err != nil { - return nil, err + return err } value := int32(v) + last tri[j] = value @@ -87,6 +162,24 @@ } } + return nil +} + +func loadReader(r *bufio.Reader) (*Tree, error) { + tree := new(Tree) + + var tin Tin + + if err := tin.Deserialize(r); err != nil { + return nil, err + } + + tree.EPSG = tin.EPSG + tree.vertices = tin.Vertices + tree.triangles = tin.Triangles + tree.Min = tin.Min + tree.Max = tin.Max + var numNodes uint32 if err := binary.Read(r, binary.LittleEndian, &numNodes); err != nil { return nil, err @@ -97,7 +190,7 @@ tree.index = make([]int32, numNodes) entries := tree.index[1:] - last = 0 + var last int32 for i := range entries { v, err := binary.ReadVarint(r) if err != nil {
--- a/pkg/octree/strtree.go Mon Oct 14 08:25:13 2019 +0200 +++ b/pkg/octree/strtree.go Mon Oct 14 12:26:00 2019 +0200 @@ -14,6 +14,11 @@ package octree import ( + "bytes" + "compress/gzip" + "encoding/binary" + "io" + "log" "math" "sort" ) @@ -153,6 +158,94 @@ return removed } +func (s *STRTree) serializeIndex(w io.Writer) error { + + if err := binary.Write(w, binary.LittleEndian, int32(len(s.index))); err != nil { + return err + } + + var buf [binary.MaxVarintLen32]byte + + var last int32 + var written int + + for _, x := range s.index { + delta := x - last + n := binary.PutVarint(buf[:], int64(delta)) + for p := buf[:n]; len(p) > 0; p = p[n:] { + var err error + if n, err = w.Write(p); err != nil { + return err + } + written += n + } + last = x + } + + log.Printf("info: compressed index in bytes: %d %.2f (%d %.2f)\n", + written, + float64(written)/(1024*1024), + 4*len(s.index), + float64(4*len(s.index))/(1024*1024), + ) + + return nil +} + +func (s *STRTree) serializeBBoxes(w io.Writer) (rr error) { + + if err := binary.Write(w, binary.LittleEndian, int32(len(s.bboxes))); err != nil { + return err + } + + var err error + + write := func(v float64) { + if err == nil { + err = binary.Write(w, binary.LittleEndian, math.Float64bits(v)) + } + } + for _, box := range s.bboxes { + write(box.X1) + write(box.Y1) + write(box.X2) + write(box.Y2) + } + + return err +} + +func (s *STRTree) Bytes() ([]byte, error) { + + var buf bytes.Buffer + w, err := gzip.NewWriterLevel(&buf, gzip.BestSpeed) + if err != nil { + return nil, err + } + + if err := s.tin.serialize(w); err != nil { + return nil, err + } + + if err := binary.Write(w, binary.LittleEndian, uint8(s.Entries)); err != nil { + return nil, err + } + + if err := s.serializeIndex(w); err != nil { + return nil, err + } + + if err := s.serializeBBoxes(w); err != nil { + return nil, err + } + + if err := w.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + func (s *STRTree) allTriangles(pos int32, tris map[int32]struct{}) { stack := []int32{pos} for len(stack) > 0 {
--- a/pkg/octree/tin.go Mon Oct 14 08:25:13 2019 +0200 +++ b/pkg/octree/tin.go Mon Oct 14 12:26:00 2019 +0200 @@ -224,10 +224,21 @@ return err } + var err error + vwrite := func(v float64) { + if err == nil { + err = binary.Write(w, binary.LittleEndian, math.Float64bits(v)) + } + } + for _, v := range t.Vertices { - if err := v.Write(w); err != nil { - return err - } + vwrite(v.X) + vwrite(v.Y) + vwrite(v.Z) + } + + if err != nil { + return err } log.Printf("info: vertices %d (%d)\n", len(t.Vertices), len(t.Vertices)*3*8)
--- a/pkg/octree/tree.go Mon Oct 14 08:25:13 2019 +0200 +++ b/pkg/octree/tree.go Mon Oct 14 12:26:00 2019 +0200 @@ -52,6 +52,50 @@ {0.5, 0.5, 1.0, 1.0}, } +func (ot *Tree) Tin() *Tin { + return &Tin{ + EPSG: ot.EPSG, + Vertices: ot.vertices, + Triangles: ot.triangles, + Min: ot.Min, + Max: ot.Max, + } +} + +func (ot *Tree) FindUnused() map[int32]struct{} { + + used := make(map[int32]struct{}, len(ot.triangles)) + for i := int32(0); i < int32(len(ot.triangles)); i++ { + used[i] = struct{}{} + } + + stack := []int32{1} + + for len(stack) > 0 { + top := stack[len(stack)-1] + stack = stack[:len(stack)-1] + + if top > 0 { // node + if index := ot.index[top:]; len(index) > 7 { + for _, idx := range index[:8] { + if idx != 0 { + stack = append(stack, idx) + } + } + } + } else { // leaf + pos := -top - 1 + n := ot.index[pos] + indices := ot.index[pos+1 : pos+1+n] + for _, idx := range indices { + delete(used, idx) + } + } + } + + return used +} + func (ot *Tree) Value(x, y float64) (float64, bool) { // out of bounding box
--- a/schema/gemma.sql Mon Oct 14 08:25:13 2019 +0200 +++ b/schema/gemma.sql Mon Oct 14 12:26:00 2019 +0200 @@ -712,6 +712,8 @@ depth_reference varchar NOT NULL, -- REFERENCES depth_references, octree_checksum varchar, octree_index bytea, + mesh_checksum varchar, + mesh_index bytea, staging_done boolean NOT NULL DEFAULT false ) CREATE CONSTRAINT TRIGGER a_sounding_results_reference_bottleneck