# HG changeset patch # User Sascha L. Teichmann # Date 1538489117 -7200 # Node ID 66e2b038d336e5ca146eb16c32ce6cf99cd42376 # Parent 0a563fef64a9259f62a4bef10979f071eb2fbcdd# Parent b3c18ced286c5d2ecfc8b874df9aa6fc13fec89a Merged default into geo-style branch. diff -r b3c18ced286c -r 66e2b038d336 cmd/gemma/main.go --- a/cmd/gemma/main.go Tue Oct 02 16:03:24 2018 +0200 +++ b/cmd/gemma/main.go Tue Oct 02 16:05:17 2018 +0200 @@ -39,7 +39,7 @@ prepareSessionStore() // Do GeoServer setup in background. - go geoserver.ConfigureBoot() + geoserver.Reconfigure(geoserver.PrepareGeoServer) m := mux.NewRouter() controllers.BindRoutes(m) diff -r b3c18ced286c -r 66e2b038d336 pkg/auth/opendb.go --- a/pkg/auth/opendb.go Tue Oct 02 16:03:24 2018 +0200 +++ b/pkg/auth/opendb.go Tue Oct 02 16:05:17 2018 +0200 @@ -4,6 +4,7 @@ "context" "database/sql" "errors" + "net/http" "sync" "github.com/jackc/pgx" @@ -12,7 +13,10 @@ "gemma.intevation.de/gemma/pkg/config" ) -var ErrNoMetamorphUser = errors.New("No metamorphic user configured") +var ( + ErrNoMetamorphUser = errors.New("No metamorphic user configured") + ErrNotLoggedIn = errors.New("Not logged in") +) func OpenDB(user, password string) (*sql.DB, error) { @@ -117,3 +121,15 @@ defer conn.Close() return fn(conn) } + +func RunAsSessionUser(req *http.Request, fn func(*sql.Conn) error) error { + token, ok := GetToken(req) + if !ok { + return ErrNotLoggedIn + } + session := Sessions.Session(token) + if session == nil { + return ErrNotLoggedIn + } + return RunAs(session.User, req.Context(), fn) +} diff -r b3c18ced286c -r 66e2b038d336 pkg/config/config.go --- a/pkg/config/config.go Tue Oct 02 16:03:24 2018 +0200 +++ b/pkg/config/config.go Tue Oct 02 16:05:17 2018 +0200 @@ -45,6 +45,7 @@ func GeoServerURL() string { return viper.GetString("geoserver-url") } func GeoServerUser() string { return viper.GetString("geoserver-user") } func GeoServerPassword() string { return viper.GetString("geoserver-password") } +func GeoServerClean() bool { return viper.GetBool("geoserver-clean") } var ( proxyKeyOnce sync.Once @@ -116,6 +117,10 @@ fl.StringSlice(name, value, usage) vbind(name) } + bl := func(name string, value bool, usage string) { + fl.Bool(name, value, usage) + vbind(name) + } strP("db-host", "H", "localhost", "host of the database") uiP("db-port", "P", 5432, "port of the database") @@ -143,6 +148,7 @@ str("geoserver-url", "http://localhost:8080/geoserver", "URL to GeoServer") str("geoserver-user", "admin", "GeoServer user") str("geoserver-password", "geoserver", "GeoServer password") + bl("geoserver-clean", false, "Clean GeoServer setup") str("proxy-key", "", `signing key for proxy URLs. Defaults to random key.`) str("proxy-prefix", "", `URL prefix of proxy. Defaults to "http://${web-host}:${web-port}"`) diff -r b3c18ced286c -r 66e2b038d336 pkg/controllers/geostyling.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/controllers/geostyling.go Tue Oct 02 16:05:17 2018 +0200 @@ -0,0 +1,68 @@ +package controllers + +import ( + "bytes" + "fmt" + "io" + "log" + "net/http" + + "gemma.intevation.de/gemma/pkg/models" + "github.com/gorilla/mux" +) + +const ( + maxStyleSize = 5 * 1024 * 1024 + styleName = "style" +) + +func extractStyle(req *http.Request) (string, error) { + + f, _, err := req.FormFile(styleName) + if err != nil { + return "", err + } + defer f.Close() + + var buf bytes.Buffer + + if _, err := io.Copy(&buf, io.LimitReader(f, maxStyleSize)); err != nil { + return "", err + } + return buf.String(), nil +} + +func supportedWMSFeature(name string) bool { + return len(models.InternalServices.Filter( + models.IntAnd(models.IntWMS, models.IntByName(name)))) > 0 +} + +func uploadStyle(rw http.ResponseWriter, req *http.Request) { + + feature := mux.Vars(req)["feature"] + + // only allow internal WMS features + if !supportedWMSFeature(feature) { + http.Error(rw, + fmt.Sprintf("WMS feature %s is not found.", feature), + http.StatusNotFound) + return + } + + style, err := extractStyle(req) + if err != nil { + log.Printf("error: %v\n", err) + http.Error(rw, "error: "+err.Error(), http.StatusBadRequest) + return + } + + log.Printf("uploaded file length: %d\n", len(style)) + + if err := models.UpdateInternalStyle(req, feature, style); err != nil { + log.Printf("error: %v\n", err) + http.Error(rw, "error: "+err.Error(), http.StatusInternalServerError) + return + } + + // TODO: Configure GeoServer +} diff -r b3c18ced286c -r 66e2b038d336 pkg/controllers/routes.go --- a/pkg/controllers/routes.go Tue Oct 02 16:03:24 2018 +0200 +++ b/pkg/controllers/routes.go Tue Oct 02 16:05:17 2018 +0200 @@ -124,7 +124,6 @@ })).Methods(http.MethodGet) // Cross sections - api.Handle("/cross", any(&JSONHandler{ Input: func() interface{} { return new(models.CrossSectionInput) }, Handle: crossSection, @@ -136,6 +135,10 @@ Handle: searchFeature, })).Methods(http.MethodPost) + // Geo styling + api.Handle("/geo/style/{feature}", + sysAdmin(http.HandlerFunc(uploadStyle))).Methods(http.MethodPost) + // Token handling: Login/Logout. api.HandleFunc("/login", login). Methods(http.MethodPost) diff -r b3c18ced286c -r 66e2b038d336 pkg/geoserver/boot.go --- a/pkg/geoserver/boot.go Tue Oct 02 16:03:24 2018 +0200 +++ b/pkg/geoserver/boot.go Tue Oct 02 16:05:17 2018 +0200 @@ -3,13 +3,13 @@ import ( "bytes" "encoding/json" + "encoding/xml" "fmt" + "io" "log" - "net" "net/http" "net/url" "strings" - "time" "gemma.intevation.de/gemma/pkg/config" "gemma.intevation.de/gemma/pkg/models" @@ -33,13 +33,27 @@ } } +func toStream(x interface{}) io.Reader { + var buf bytes.Buffer + + if err := json.NewEncoder(&buf).Encode(x); err != nil { + // Should not happen + log.Printf("bad JSON: %v\n", err) + } + return bytes.NewReader(buf.Bytes()) +} + func asJSON(req *http.Request) { req.Header.Set("Content-Type", "application/json") } +func asContentType(req *http.Request, contentType string) { + req.Header.Set("Content-Type", contentType) +} + func ensureWorkspace() error { var ( - url = config.GeoServerURL() + geoURL = config.GeoServerURL() user = config.GeoServerUser() password = config.GeoServerPassword() auth = basicAuth(user, password) @@ -48,7 +62,7 @@ // Probe workspace. req, err := http.NewRequest( http.MethodGet, - url+"/rest/workspaces/"+workspaceName+".json", + geoURL+"/rest/workspaces/"+workspaceName+".json", nil) if err != nil { return err @@ -72,7 +86,7 @@ req, err = http.NewRequest( http.MethodPost, - url+"/rest/workspaces", + geoURL+"/rest/workspaces", strings.NewReader(createJSON)) if err != nil { return err @@ -94,7 +108,7 @@ func ensureDataStore() error { var ( - url = config.GeoServerURL() + geoURL = config.GeoServerURL() user = config.GeoServerUser() password = config.GeoServerPassword() auth = basicAuth(user, password) @@ -103,7 +117,8 @@ // Probe datastore. req, err := http.NewRequest( http.MethodGet, - url+"/rest/workspaces/"+workspaceName+"/datastores/"+datastoreName+".json", + geoURL+"/rest/workspaces/"+workspaceName+ + "/datastores/"+datastoreName+".json", nil) if err != nil { return err @@ -146,16 +161,11 @@ }, }, } - var out bytes.Buffer - enc := json.NewEncoder(&out) - if err := enc.Encode(&ds); err != nil { - return err - } req, err = http.NewRequest( http.MethodPost, - url+"/rest/workspaces/"+workspaceName+"/datastores", - bytes.NewReader(out.Bytes())) + geoURL+"/rest/workspaces/"+workspaceName+"/datastores", + toStream(ds)) if err != nil { return err } @@ -177,7 +187,7 @@ func ensureFeatures() error { var ( - url = config.GeoServerURL() + geoURL = config.GeoServerURL() user = config.GeoServerUser() password = config.GeoServerPassword() auth = basicAuth(user, password) @@ -211,7 +221,7 @@ // Fetch all featuretypes. req, err := http.NewRequest( http.MethodGet, - url+"/rest/workspaces/"+workspaceName+ + geoURL+"/rest/workspaces/"+workspaceName+ "/datastores/"+datastoreName+ "/featuretypes.json", nil) @@ -255,18 +265,12 @@ }, } - var out bytes.Buffer - enc := json.NewEncoder(&out) - if err := enc.Encode(&ft); err != nil { - return err - } - req, err := http.NewRequest( http.MethodPost, - url+"/rest/workspaces/"+workspaceName+ + geoURL+"/rest/workspaces/"+workspaceName+ "/datastores/"+datastoreName+ "/featuretypes", - bytes.NewReader(out.Bytes())) + toStream(ft)) if err != nil { return err } @@ -287,7 +291,205 @@ return nil } -func prepareGeoServer() error { +func deleteWorkspace() error { + + // Should we delete our workspace first? + if !config.GeoServerClean() { + return nil + } + + log.Println("info: delete workspace " + workspaceName) + var ( + geoURL = config.GeoServerURL() + user = config.GeoServerUser() + password = config.GeoServerPassword() + auth = basicAuth(user, password) + ) + + req, err := http.NewRequest( + http.MethodDelete, + geoURL+"/rest/workspaces/"+workspaceName+"?recurse=true", + nil) + if err != nil { + return err + } + auth(req) + _, err = http.DefaultClient.Do(req) + return err +} + +func ensureStyles() error { + log.Println("info: creating styles") + + var ( + geoURL = config.GeoServerURL() + user = config.GeoServerUser() + password = config.GeoServerPassword() + auth = basicAuth(user, password) + ) + + type styles struct { + Styles struct { + Style []struct { + Name string `json:"name"` + } `json:"style"` + } `json:"styles"` + } + + var stls styles + + hasStyle := func(name string) bool { + for i := range stls.Styles.Style { + if stls.Styles.Style[i].Name == name { + return true + } + } + return false + } + + req, err := http.NewRequest( + http.MethodGet, + geoURL+"/rest/workspaces/"+workspaceName+"/styles.json", + nil) + if err != nil { + return err + } + auth(req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + // Fetch all styles + if err := json.NewDecoder(resp.Body).Decode(&stls); err != nil { + // XXX: Same quirk as with featuretypes. + } + resp.Body.Close() + + entries := models.InternalServices.Filter( + models.IntAnd(models.IntWMS, models.IntWithStyle)) + + for i := range entries { + if hasStyle(entries[i].Name) { + log.Printf("already has style for %s\n", entries[i].Name) + continue + } + + log.Printf("creating style %s\n", entries[i].Name) + + styleURL := geoURL + "/rest/workspaces/" + workspaceName + + "/styles" + + // First create style + + type Style struct { + Name string `json:"name"` + Filename string `json:"filename"` + } + + var layer = struct { + Style Style `json:"style"` + }{ + Style: Style{ + Name: entries[i].Name, + Filename: entries[i].Name + ".sld", + }, + } + + req, err := http.NewRequest( + http.MethodPost, + styleURL, + toStream(&layer)) + + if err != nil { + return err + } + auth(req) + asJSON(req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusCreated { + return fmt.Errorf("Unable to create style %s (%s)", + entries[i].Name, + http.StatusText(resp.StatusCode)) + } + + // Second upload data + + req, err = http.NewRequest( + http.MethodPut, + styleURL+"/"+url.PathEscape(entries[i].Name), + strings.NewReader(entries[i].Style.String)) + if err != nil { + return err + } + auth(req) + if isSymbologyEncoding(entries[i].Style.String) { + asContentType(req, "application/vnd.ogc.se+xml") + } else { + asContentType(req, "application/vnd.ogc.sld+xml") + } + resp, err = http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("cannot upload style %s (%s)", + entries[i].Name, http.StatusText(resp.StatusCode)) + } + + // Third associate with layer + + req, err = http.NewRequest( + http.MethodPost, + geoURL+"/rest/layers/"+ + url.PathEscape(workspaceName+":"+entries[i].Name)+ + "/styles?default=true", + toStream(&layer)) + if err != nil { + return err + } + auth(req) + asJSON(req) + + resp, err = http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusCreated { + return fmt.Errorf("cannot connect style %s with layer (%s)", + entries[i].Name, http.StatusText(resp.StatusCode)) + } + } + + return nil +} + +// isSymbologyEncoding tries to figure out if its plain SLD or SE. +func isSymbologyEncoding(data string) bool { + decoder := xml.NewDecoder(strings.NewReader(data)) + for { + tok, err := decoder.Token() + switch { + case tok == nil && err == io.EOF: + return false + case err != nil: + log.Printf("warn: invalid XML: %v\n", err) + return false + } + if t, ok := tok.(xml.StartElement); ok && + t.Name.Space == "http://www.opengis.net/se" { + return true + } + } +} + +func PrepareGeoServer() error { if config.DBUser() == "" { log.Println("info: Need metamorphic db user to configure GeoServer") @@ -295,44 +497,21 @@ } if config.GeoServerURL() == "" { - log.Println("info: No tables to publish on GeoServer") + log.Println("info: No URL to GeoServer configured") return nil } - if err := ensureWorkspace(); err != nil { - return err - } - - if err := ensureDataStore(); err != nil { - return err + for _, ensure := range []func() error{ + deleteWorkspace, + ensureWorkspace, + ensureDataStore, + ensureFeatures, + ensureStyles, + } { + if err := ensure(); err != nil { + return err + } } - // TODO: Styles - - return ensureFeatures() + return nil } - -func ConfigureBoot() { - log.Println("Configure GeoServer...") - const maxTries = 10 - const sleep = time.Second * 5 - - for try := 1; try <= maxTries; try++ { - err := prepareGeoServer() - if err == nil { - break - } - if try < maxTries { - if uerr, ok := err.(*url.Error); ok { - if oerr, ok := uerr.Err.(*net.OpError); ok && oerr.Op == "dial" { - log.Printf("Failed attempt %d of %d to configure GeoServer. "+ - "Will try again in %s...\n", try, maxTries, sleep) - time.Sleep(sleep) - continue - } - } - } - log.Printf("warn: configure GeoServer failed: %v\n", err) - break - } -} diff -r b3c18ced286c -r 66e2b038d336 pkg/geoserver/reconf.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/geoserver/reconf.go Tue Oct 02 16:05:17 2018 +0200 @@ -0,0 +1,67 @@ +package geoserver + +import ( + "container/list" + "log" + "net" + "net/url" + "sync" + "time" +) + +var ( + confQueue = list.New() + confQueueCond = sync.NewCond(new(sync.Mutex)) +) + +func init() { + go asyncConfigure() +} + +func asyncConfigure() { + for { + var fn func() error + confQueueCond.L.Lock() + for confQueue.Len() == 0 { + confQueueCond.Wait() + } + fn = confQueue.Remove(confQueue.Front()).(func() error) + confQueueCond.L.Unlock() + if err := reconfigure(fn); err != nil { + log.Printf("warn: configure GeoServer failed: %v\n", err) + } + } +} + +func reconfigure(fn func() error) error { + log.Println("Configure GeoServer...") + const ( + maxTries = 10 + sleep = time.Second * 5 + ) + var err error + for try := 1; try <= maxTries; try++ { + if err = fn(); err == nil { + break + } + if try < maxTries { + if uerr, ok := err.(*url.Error); ok { + if oerr, ok := uerr.Err.(*net.OpError); ok && oerr.Op == "dial" { + log.Printf("Failed attempt %d of %d to configure GeoServer. "+ + "Will try again in %s...\n", try, maxTries, sleep) + time.Sleep(sleep) + continue + } + } + } + break + } + return err +} + +func Reconfigure(fn func() error) { + confQueueCond.L.Lock() + defer confQueueCond.L.Unlock() + confQueue.PushBack(fn) + confQueueCond.Signal() +} diff -r b3c18ced286c -r 66e2b038d336 pkg/models/intservices.go --- a/pkg/models/intservices.go Tue Oct 02 16:03:24 2018 +0200 +++ b/pkg/models/intservices.go Tue Oct 02 16:05:17 2018 +0200 @@ -4,6 +4,7 @@ "context" "database/sql" "log" + "net/http" "sync" "gemma.intevation.de/gemma/pkg/auth" @@ -22,11 +23,30 @@ mu sync.Mutex } -const selectPublishedServices = `SELECT relname, style, as_wms, as_wfs +const ( + selectPublishedServices = `SELECT relname, style, as_wms, as_wfs FROM sys_admin.published_services JOIN pg_class ON name = oid ORDER by relname` + updateStyleSQL = ` +UPDATE sys_admin.published_services +SET style = $1::bytea +WHERE name IN (SELECT oid FROM pg_class WHERE relname = $2)` +) + var InternalServices = &IntServices{} +func UpdateInternalStyle(req *http.Request, name, style string) error { + return auth.RunAsSessionUser(req, func(conn *sql.Conn) error { + _, err := conn.ExecContext( + req.Context(), updateStyleSQL, + style, name) + if err == nil { + InternalServices.Invalidate() + } + return err + }) +} + func (ps *IntServices) Find(name string) (string, bool) { ps.mu.Lock() defer ps.mu.Unlock() @@ -97,6 +117,18 @@ func IntWMS(entry IntEntry) bool { return entry.WMS } func IntWFS(entry IntEntry) bool { return entry.WFS } +func IntByName(name string) func(IntEntry) bool { + return func(entry IntEntry) bool { return entry.Name == name } +} + +func IntAnd(a, b func(IntEntry) bool) func(IntEntry) bool { + return func(entry IntEntry) bool { return a(entry) && b(entry) } +} + +func IntWithStyle(entry IntEntry) bool { + return entry.Style.Valid +} + func (ps *IntServices) Filter(accept func(IntEntry) bool) []IntEntry { ps.mu.Lock() defer ps.mu.Unlock() diff -r b3c18ced286c -r 66e2b038d336 schema/auth.sql --- a/schema/auth.sql Tue Oct 02 16:03:24 2018 +0200 +++ b/schema/auth.sql Tue Oct 02 16:05:17 2018 +0200 @@ -33,6 +33,7 @@ GRANT SELECT ON ALL TABLES IN SCHEMA sys_admin TO sys_admin; GRANT UPDATE ON sys_admin.system_config TO sys_admin; GRANT UPDATE ON systemconf.feature_colours TO sys_admin; +GRANT UPDATE ON sys_admin.published_services TO sys_admin; GRANT INSERT, DELETE ON sys_admin.password_reset_requests TO sys_admin; GRANT INSERT, DELETE, UPDATE ON waterway.sounding_results_contour_lines TO sys_admin; diff -r b3c18ced286c -r 66e2b038d336 schema/gemma.sql