gemma: cmd/soundingresults/main.go annotate

annotate cmd/soundingresults/main.go @ 603:3d33c53db1e3

Sounding results: Read point data from xyz files.

author	Sascha L. Teichmann <sascha.teichmann@intevation.de>
date	Mon, 10 Sep 2018 12:10:10 +0200
parents	10f898bbe50f
children	4d97066c311c

rev	line source
600 30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	1 package main
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	2
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	3 import (
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	4 "bufio"
603 3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	5 "compress/bzip2"
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	6 "compress/gzip"
600 30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	7 "database/sql"
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	8 "flag"
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	9 "fmt"
603 3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	10 "io"
600 30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	11 "log"
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	12 "os"
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	13 "path/filepath"
603 3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	14 "runtime"
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	15 "strconv"
600 30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	16 "strings"
603 3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	17 "sync"
600 30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	18 "time"
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	19
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	20 "github.com/jackc/pgx"
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	21 "github.com/jackc/pgx/stdlib"
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	22 )
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	23
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	24 var (
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	25 insecure = flag.Bool("insecure", false, "skip SSL verification")
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	26 dbhost = flag.String("dbhost", "localhost", "database host")
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	27 dbport = flag.Uint("dbport", 5432, "database port")
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	28 dbname = flag.String("dbname", "gemma", "database user")
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	29 dbuser = flag.String("dbuser", "scott", "database user")
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	30 dbpassword = flag.String("dbpw", "tiger", "database password")
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	31 dbssl = flag.String("dbssl", "prefer", "database SSL mode")
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	32 )
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	33
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	34 func run(fn func(*sql.DB) error) error {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	35
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	36 // To ease SSL config ride a bit on parsing.
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	37 cc, err := pgx.ParseConnectionString("sslmode=" + *dbssl)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	38 if err != nil {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	39 return err
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	40 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	41
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	42 // Do the rest manually to allow whitespace in user/password.
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	43 cc.Host = *dbhost
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	44 cc.Port = uint16(*dbport)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	45 cc.User = *dbuser
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	46 cc.Password = *dbpassword
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	47 cc.Database = *dbname
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	48
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	49 db := stdlib.OpenDB(cc)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	50 defer db.Close()
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	51
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	52 return fn(db)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	53 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	54
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	55 type meta struct {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	56 date time.Time
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	57 name string
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	58 depthReference string
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	59 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	60
603 3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	61 func wrap(fname string, f io.Reader) (io.Reader, error) {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	62
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	63 switch strings.ToLower(filepath.Ext(fname)) {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	64 case ".gz":
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	65 return gzip.NewReader(f)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	66 case ".bz2":
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	67 return bzip2.NewReader(f), nil
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	68 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	69
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	70 return bufio.NewReader(f), nil
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	71 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	72
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	73 type point3d struct {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	74 x float64
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	75 y float64
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	76 z float64
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	77 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	78
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	79 func parseXYZ(fname string) ([]*point3d, error) {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	80 f, err := os.Open(fname)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	81 if err != nil {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	82 return nil, err
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	83 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	84 defer f.Close()
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	85
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	86 r, err := wrap(fname, f)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	87 if err != nil {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	88 return nil, err
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	89 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	90
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	91 // Alloc in larger chunks to reduce pressure on memory management.
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	92 var chunk []point3d
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	93 alloc := func() *point3d {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	94 if len(chunk) == 0 {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	95 chunk = make([]point3d, 8*1024)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	96 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	97 p := &chunk[0]
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	98 chunk = chunk[1:]
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	99 return p
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	100 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	101
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	102 var points []*point3d
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	103
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	104 s := bufio.NewScanner(r)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	105 if s.Scan() { // Skip header line.
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	106 for line := 2; s.Scan(); line++ {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	107 p := alloc()
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	108 text := s.Text()
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	109 // fmt.Sscanf(text, "%f,%f,%f") is 4 times slower.
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	110 idx := strings.IndexByte(text, ',')
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	111 if idx == -1 {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	112 log.Printf("format error in line %d\n", line)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	113 continue
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	114 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	115 if p.x, err = strconv.ParseFloat(text[:idx], 64); err != nil {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	116 log.Printf("format error in line %d: %v\n", line, err)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	117 continue
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	118 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	119 text = text[idx+1:]
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	120 if idx = strings.IndexByte(text, ','); idx == -1 {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	121 log.Printf("format error in line %d\n", line)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	122 continue
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	123 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	124 if p.y, err = strconv.ParseFloat(text[:idx], 64); err != nil {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	125 log.Printf("format error in line %d: %v\n", line, err)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	126 continue
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	127 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	128 text = text[idx+1:]
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	129 if p.z, err = strconv.ParseFloat(text, 64); err != nil {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	130 log.Printf("format error in line %d: %v\n", line, err)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	131 continue
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	132 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	133 points = append(points, p)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	134 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	135 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	136
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	137 return points, s.Err()
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	138 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	139
600 30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	140 func substituteName(fname, name string) string {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	141 dir := filepath.Dir(fname)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	142 info := filepath.Join(dir, "INFO.txt")
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	143 f, err := os.Open(info)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	144 if err != nil {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	145 log.Printf("warn: %v\n", err)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	146 return name
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	147 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	148 defer f.Close()
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	149
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	150 s := bufio.NewScanner(f)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	151
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	152 for search := strings.ToLower(name); s.Scan(); {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	153 line := strings.TrimSpace(s.Text())
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	154 if line == "" \|\| strings.HasPrefix(line, "#") {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	155 continue
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	156 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	157
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	158 if parts := strings.SplitN(line, "=", 2); len(parts) == 2 &&
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	159 strings.TrimSpace(strings.ToLower(parts[0])) == search {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	160 return strings.TrimSpace(parts[1])
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	161 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	162 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	163
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	164 if err := s.Err(); err != nil {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	165 log.Printf("error: %v\n", err)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	166 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	167
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	168 return name
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	169 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	170
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	171 func parseFilename(fname string) (meta, error) {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	172
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	173 base := filepath.Base(fname)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	174
602 10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	175 compressed := strings.ToLower(filepath.Ext(base))
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	176 for _, ext := range []string{".gz", ".bz2"} {
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	177 if ext == compressed {
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	178 base = base[:len(base)-len(ext)]
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	179 break
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	180 }
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	181 }
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	182
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	183 // Cut .txt
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	184 base = base[:len(base)-len(filepath.Ext(base))]
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	185
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	186 if !strings.HasSuffix(strings.ToUpper(base), "_WGS84") {
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	187 return meta{}, fmt.Errorf("%s is not in WGS84", base)
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	188 }
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	189
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	190 base = base[:len(base)-len("_WGS84")]
10f898bbe50f Sounding results: Accept if XYZ files are compressed with GZIP or BZIP2 Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 600 diff changeset	191
600 30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	192 idx := strings.IndexRune(base, '_')
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	193 if idx == -1 {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	194 return meta{}, fmt.Errorf("%s has no date", base)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	195 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	196
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	197 datePart := base[:idx]
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	198
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	199 date, err := time.Parse("20060102", datePart)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	200 if err != nil {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	201 return meta{}, fmt.Errorf("error %s: %v\n", err)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	202 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	203
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	204 rest := base[idx+1:]
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	205
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	206 if idx = strings.LastIndex(rest, "_"); idx == -1 {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	207 return meta{}, fmt.Errorf("%s has no depth reference", base)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	208 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	209
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	210 depthReference := rest[idx+1:]
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	211
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	212 rest = rest[:idx]
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	213
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	214 if !strings.HasSuffix(strings.ToUpper(rest), "_MB") {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	215 return meta{}, fmt.Errorf("%s is not in WGS84", base)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	216 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	217
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	218 name := rest[:len(rest)-len("_MB")]
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	219
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	220 name = substituteName(fname, name)
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	221
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	222 return meta{
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	223 name: name,
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	224 depthReference: depthReference,
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	225 date: date,
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	226 }, nil
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	227 }
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	228
603 3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	229 func processor(fnames <-chan string, wg *sync.WaitGroup) {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	230 defer wg.Done()
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	231
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	232 for fname := range fnames {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	233 log.Printf("Processing %s\n", fname)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	234 m, err := parseFilename(fname)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	235 if err != nil {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	236 log.Printf("error: %v\n", err)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	237 continue
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	238 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	239 _ = m
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	240 points, err := parseXYZ(fname)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	241 if err != nil {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	242 log.Printf("error: %v\n", err)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	243 continue
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	244 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	245 fmt.Printf("Number of points: %d\n", len(points))
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	246 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	247 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	248
600 30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	249 func main() {
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	250 flag.Parse()
30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	251
603 3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	252 var wg sync.WaitGroup
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	253
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	254 fnames := make(chan string)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	255
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	256 for i, n := 0, runtime.NumCPU(); i < n; i++ {
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	257 wg.Add(1)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	258 go processor(fnames, &wg)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	259 }
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	260
600 30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	261 for _, fname := range flag.Args() {
603 3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	262 fnames <- fname
600 30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	263 }
603 3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	264
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	265 close(fnames)
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	266
3d33c53db1e3 Sounding results: Read point data from xyz files. Sascha L. Teichmann <sascha.teichmann@intevation.de> parents: 602 diff changeset	267 wg.Wait()
600 30eb8c823ff7 Started with importing the sounding results. WIP. Sascha L. Teichmann <teichmann@intevation.de> parents: diff changeset	268 }

Mercurial > gemma

annotate cmd/soundingresults/main.go @ 603:3d33c53db1e3