Mercurial > gemma

comparison cmd/soundingresults/main.go @ 603:3d33c53db1e3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Sounding results: Read point data from xyz files.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Mon, 10 Sep 2018 12:10:10 +0200
parents 10f898bbe50f
children 4d97066c311c
comparison
equal deleted inserted replaced
602:10f898bbe50f 603:3d33c53db1e3
1 package main 1 package main
2 2
3 import ( 3 import (
4 "bufio" 4 "bufio"
5 "compress/bzip2"
6 "compress/gzip"
5 "database/sql" 7 "database/sql"
6 "flag" 8 "flag"
7 "fmt" 9 "fmt"
10 "io"
8 "log" 11 "log"
9 "os" 12 "os"
10 "path/filepath" 13 "path/filepath"
14 "runtime"
15 "strconv"
11 "strings" 16 "strings"
17 "sync"
12 "time" 18 "time"
13 19
14 "github.com/jackc/pgx" 20 "github.com/jackc/pgx"
15 "github.com/jackc/pgx/stdlib" 21 "github.com/jackc/pgx/stdlib"
16 ) 22 )
50 date time.Time 56 date time.Time
51 name string 57 name string
52 depthReference string 58 depthReference string
53 } 59 }
54 60
61 func wrap(fname string, f io.Reader) (io.Reader, error) {
62
63 switch strings.ToLower(filepath.Ext(fname)) {
64 case ".gz":
65 return gzip.NewReader(f)
66 case ".bz2":
67 return bzip2.NewReader(f), nil
68 }
69
70 return bufio.NewReader(f), nil
71 }
72
73 type point3d struct {
74 x float64
75 y float64
76 z float64
77 }
78
79 func parseXYZ(fname string) ([]*point3d, error) {
80 f, err := os.Open(fname)
81 if err != nil {
82 return nil, err
83 }
84 defer f.Close()
85
86 r, err := wrap(fname, f)
87 if err != nil {
88 return nil, err
89 }
90
91 // Alloc in larger chunks to reduce pressure on memory management.
92 var chunk []point3d
93 alloc := func() *point3d {
94 if len(chunk) == 0 {
95 chunk = make([]point3d, 8*1024)
96 }
97 p := &chunk[0]
98 chunk = chunk[1:]
99 return p
100 }
101
102 var points []*point3d
103
104 s := bufio.NewScanner(r)
105 if s.Scan() { // Skip header line.
106 for line := 2; s.Scan(); line++ {
107 p := alloc()
108 text := s.Text()
109 // fmt.Sscanf(text, "%f,%f,%f") is 4 times slower.
110 idx := strings.IndexByte(text, ',')
111 if idx == -1 {
112 log.Printf("format error in line %d\n", line)
113 continue
114 }
115 if p.x, err = strconv.ParseFloat(text[:idx], 64); err != nil {
116 log.Printf("format error in line %d: %v\n", line, err)
117 continue
118 }
119 text = text[idx+1:]
120 if idx = strings.IndexByte(text, ','); idx == -1 {
121 log.Printf("format error in line %d\n", line)
122 continue
123 }
124 if p.y, err = strconv.ParseFloat(text[:idx], 64); err != nil {
125 log.Printf("format error in line %d: %v\n", line, err)
126 continue
127 }
128 text = text[idx+1:]
129 if p.z, err = strconv.ParseFloat(text, 64); err != nil {
130 log.Printf("format error in line %d: %v\n", line, err)
131 continue
132 }
133 points = append(points, p)
134 }
135 }
136
137 return points, s.Err()
138 }
139
55 func substituteName(fname, name string) string { 140 func substituteName(fname, name string) string {
56 dir := filepath.Dir(fname) 141 dir := filepath.Dir(fname)
57 info := filepath.Join(dir, "INFO.txt") 142 info := filepath.Join(dir, "INFO.txt")
58 f, err := os.Open(info) 143 f, err := os.Open(info)
59 if err != nil { 144 if err != nil {
102 return meta{}, fmt.Errorf("%s is not in WGS84", base) 187 return meta{}, fmt.Errorf("%s is not in WGS84", base)
103 } 188 }
104 189
105 base = base[:len(base)-len("_WGS84")] 190 base = base[:len(base)-len("_WGS84")]
106 191
107 log.Printf("base: %s\n", base)
108
109 idx := strings.IndexRune(base, '_') 192 idx := strings.IndexRune(base, '_')
110 if idx == -1 { 193 if idx == -1 {
111 return meta{}, fmt.Errorf("%s has no date", base) 194 return meta{}, fmt.Errorf("%s has no date", base)
112 } 195 }
113 196
141 depthReference: depthReference, 224 depthReference: depthReference,
142 date: date, 225 date: date,
143 }, nil 226 }, nil
144 } 227 }
145 228
229 func processor(fnames <-chan string, wg *sync.WaitGroup) {
230 defer wg.Done()
231
232 for fname := range fnames {
233 log.Printf("Processing %s\n", fname)
234 m, err := parseFilename(fname)
235 if err != nil {
236 log.Printf("error: %v\n", err)
237 continue
238 }
239 _ = m
240 points, err := parseXYZ(fname)
241 if err != nil {
242 log.Printf("error: %v\n", err)
243 continue
244 }
245 fmt.Printf("Number of points: %d\n", len(points))
246 }
247 }
248
146 func main() { 249 func main() {
147 flag.Parse() 250 flag.Parse()
148 251
252 var wg sync.WaitGroup
253
254 fnames := make(chan string)
255
256 for i, n := 0, runtime.NumCPU(); i < n; i++ {
257 wg.Add(1)
258 go processor(fnames, &wg)
259 }
260
149 for _, fname := range flag.Args() { 261 for _, fname := range flag.Args() {
150 m, err := parseFilename(fname) 262 fnames <- fname
151 if err != nil { 263 }
152 log.Printf("error: %s\n", err) 264
153 continue 265 close(fnames)
154 } 266
155 fmt.Printf("%q\n", m) 267 wg.Wait()
156 } 268 }
157 }