changeset 2470:5bd3236f2b1f octree-diff

Integrate deduplication with setting in. Even if there is more contention in total it seems to be cheaper.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Tue, 26 Feb 2019 11:01:06 +0100
parents 1312953defab
children 63475c8e710e
files cmd/octreediff/main.go
diffstat 1 files changed, 34 insertions(+), 35 deletions(-) [+]
line wrap: on
line diff
--- a/cmd/octreediff/main.go	Mon Feb 25 23:43:41 2019 +0100
+++ b/cmd/octreediff/main.go	Tue Feb 26 11:01:06 2019 +0100
@@ -49,11 +49,16 @@
 	}
 }
 
+type point struct {
+	x float64
+	y float64
+}
+
 func sliceWork(
 	vs []octree.Vertex,
+	dst map[point]float64,
 	fn func([]octree.Vertex, func([]octree.Vertex) []octree.Vertex),
-) []octree.Vertex {
-
+) {
 	n := runtime.NumCPU()
 
 	wg := new(sync.WaitGroup)
@@ -86,14 +91,21 @@
 			}
 		}()
 	}
-	nvs := make([]octree.Vertex, 0, len(vs))
 	done := make(chan struct{})
 	go func() {
 		defer close(done)
-		for ns := range out {
-			nvs = append(nvs, ns...)
+		for s := range out {
+			for i := range s {
+				v := &s[i]
+				key := point{x: v.X, y: v.Y}
+				if z, found := dst[key]; found {
+					dst[key] = (z + v.Z) * 0.5
+				} else {
+					dst[key] = v.Z
+				}
+			}
 			select {
-			case pool <- ns[:0:pageSize]:
+			case pool <- s[:0:pageSize]:
 			default:
 			}
 		}
@@ -114,7 +126,6 @@
 	wg.Wait()
 	close(out)
 	<-done
-	return nvs
 }
 
 func process(bottleneck string, firstDate, secondDate time.Time) error {
@@ -196,12 +207,17 @@
 			return errors.New("EPSG codes mismatch. Needs transformation slow pass.")
 		}
 
-		log.Printf("loading took %v\n", time.Since(start))
+		now := time.Now()
+		log.Printf("loading took %v\n", now.Sub(start))
+		last := now
+
+		firstVs, secondVs := first.Vertices(), second.Vertices()
 
-		setin := time.Now()
+		result := make(map[point]float64, len(firstVs)+len(secondVs))
 
-		nvf := sliceWork(
-			first.Vertices(),
+		sliceWork(
+			firstVs,
+			result,
 			func(
 				slice []octree.Vertex,
 				turn func([]octree.Vertex) []octree.Vertex,
@@ -221,8 +237,9 @@
 				}
 			})
 
-		nsf := sliceWork(
-			second.Vertices(),
+		sliceWork(
+			secondVs,
+			result,
 			func(
 				slice []octree.Vertex,
 				turn func([]octree.Vertex) []octree.Vertex,
@@ -242,28 +259,10 @@
 				}
 			})
 
-		/*
-			vf := first.Vertices()
-			nvf := make([]octree.Vertex, 0, len(vf))
-			for i := range vf {
-				v := &vf[i]
-				if z, found := second.Value(v.X, v.Y); found {
-					nvf = append(nvf, octree.Vertex{v.X, v.Y, v.Z - z})
-				}
-			}
-			vs := second.Vertices()
-			nsf := make([]octree.Vertex, 0, len(vs))
-			for i := range vs {
-				v := &vs[i]
-				if z, found := first.Value(v.X, v.Y); found {
-					nsf = append(nsf, octree.Vertex{v.X, v.Y, z - v.Z})
-				}
-			}
-		*/
-
-		log.Printf("setting in took %v\n", time.Since(setin))
-		log.Printf("new vertices first: %d\n", len(nvf))
-		log.Printf("new vertices second: %d\n", len(nsf))
+		now = time.Now()
+		log.Printf("setting in took %v\n", now.Sub(last))
+		last = now
+		log.Printf("num points: %d\n", len(result))
 
 		return nil
 	})