# HG changeset patch # User Sascha L. Teichmann # Date 1539003197 -7200 # Node ID 7899867c7bf5cbd19f37b6138cf7e5f609940c2c # Parent e758e12b38c9916c335edef62de94eb3bb2a60b7# Parent ae1531e00344aa3e8986787a5074f0a4ba864733 Merged default into geo-style branch. diff -r ae1531e00344 -r 7899867c7bf5 cmd/gemma/main.go --- a/cmd/gemma/main.go Mon Oct 08 14:52:37 2018 +0200 +++ b/cmd/gemma/main.go Mon Oct 08 14:53:17 2018 +0200 @@ -39,7 +39,7 @@ prepareSessionStore() // Do GeoServer setup in background. - go geoserver.ConfigureBoot() + geoserver.Reconfigure(geoserver.PrepareGeoServer) m := mux.NewRouter() controllers.BindRoutes(m) diff -r ae1531e00344 -r 7899867c7bf5 pkg/auth/opendb.go --- a/pkg/auth/opendb.go Mon Oct 08 14:52:37 2018 +0200 +++ b/pkg/auth/opendb.go Mon Oct 08 14:53:17 2018 +0200 @@ -4,6 +4,7 @@ "context" "database/sql" "errors" + "net/http" "sync" "github.com/jackc/pgx" @@ -12,7 +13,10 @@ "gemma.intevation.de/gemma/pkg/config" ) -var ErrNoMetamorphUser = errors.New("No metamorphic user configured") +var ( + ErrNoMetamorphUser = errors.New("No metamorphic user configured") + ErrNotLoggedIn = errors.New("Not logged in") +) func OpenDB(user, password string) (*sql.DB, error) { @@ -117,3 +121,15 @@ defer conn.Close() return fn(conn) } + +func RunAsSessionUser(req *http.Request, fn func(*sql.Conn) error) error { + token, ok := GetToken(req) + if !ok { + return ErrNotLoggedIn + } + session := Sessions.Session(token) + if session == nil { + return ErrNotLoggedIn + } + return RunAs(session.User, req.Context(), fn) +} diff -r ae1531e00344 -r 7899867c7bf5 pkg/config/config.go --- a/pkg/config/config.go Mon Oct 08 14:52:37 2018 +0200 +++ b/pkg/config/config.go Mon Oct 08 14:53:17 2018 +0200 @@ -45,6 +45,7 @@ func GeoServerURL() string { return viper.GetString("geoserver-url") } func GeoServerUser() string { return viper.GetString("geoserver-user") } func GeoServerPassword() string { return viper.GetString("geoserver-password") } +func GeoServerClean() bool { return viper.GetBool("geoserver-clean") } var ( proxyKeyOnce sync.Once @@ -116,6 +117,10 @@ fl.StringSlice(name, value, usage) vbind(name) } + bl := func(name string, value bool, usage string) { + fl.Bool(name, value, usage) + vbind(name) + } strP("db-host", "H", "localhost", "host of the database") uiP("db-port", "P", 5432, "port of the database") @@ -143,6 +148,7 @@ str("geoserver-url", "http://localhost:8080/geoserver", "URL to GeoServer") str("geoserver-user", "admin", "GeoServer user") str("geoserver-password", "geoserver", "GeoServer password") + bl("geoserver-clean", false, "Clean GeoServer setup") str("proxy-key", "", `signing key for proxy URLs. Defaults to random key.`) str("proxy-prefix", "", `URL prefix of proxy. Defaults to "http://${web-host}:${web-port}"`) diff -r ae1531e00344 -r 7899867c7bf5 pkg/controllers/geostyling.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/controllers/geostyling.go Mon Oct 08 14:53:17 2018 +0200 @@ -0,0 +1,75 @@ +package controllers + +import ( + "bytes" + "fmt" + "io" + "log" + "net/http" + + "github.com/gorilla/mux" + + "gemma.intevation.de/gemma/pkg/geoserver" + "gemma.intevation.de/gemma/pkg/models" +) + +const ( + maxStyleSize = 5 * 1024 * 1024 + styleName = "style" +) + +func extractStyle(req *http.Request) (string, error) { + + f, _, err := req.FormFile(styleName) + if err != nil { + return "", err + } + defer f.Close() + + var buf bytes.Buffer + + if _, err := io.Copy(&buf, io.LimitReader(f, maxStyleSize)); err != nil { + return "", err + } + return buf.String(), nil +} + +func supportedWMSFeature(name string) bool { + return len(models.InternalServices.Filter( + models.IntAnd( + models.IntWMS, + models.IntByName(name)))) > 0 +} + +func uploadStyle(rw http.ResponseWriter, req *http.Request) { + + feature := mux.Vars(req)["feature"] + + // only allow internal WMS features + if !supportedWMSFeature(feature) { + http.Error(rw, + fmt.Sprintf("WMS feature %s is not found.", feature), + http.StatusNotFound) + return + } + + style, err := extractStyle(req) + if err != nil { + log.Printf("error: %v\n", err) + http.Error(rw, "error: "+err.Error(), http.StatusBadRequest) + return + } + + log.Printf("uploaded file length: %d\n", len(style)) + + if err := models.UpdateInternalStyle(req, feature, style); err != nil { + log.Printf("error: %v\n", err) + http.Error(rw, "error: "+err.Error(), http.StatusInternalServerError) + return + } + + geoserver.ReconfigureStyle(feature) + + // Nothing to return + rw.WriteHeader(http.StatusNoContent) +} diff -r ae1531e00344 -r 7899867c7bf5 pkg/controllers/routes.go --- a/pkg/controllers/routes.go Mon Oct 08 14:52:37 2018 +0200 +++ b/pkg/controllers/routes.go Mon Oct 08 14:53:17 2018 +0200 @@ -124,7 +124,6 @@ })).Methods(http.MethodGet) // Cross sections - api.Handle("/cross", any(&JSONHandler{ Input: func() interface{} { return new(models.CrossSectionInput) }, Handle: crossSection, @@ -136,6 +135,10 @@ Handle: searchFeature, })).Methods(http.MethodPost) + // Geo styling + api.Handle("/geo/style/{feature}", + sysAdmin(http.HandlerFunc(uploadStyle))).Methods(http.MethodPost) + // Token handling: Login/Logout. api.HandleFunc("/login", login). Methods(http.MethodPost) diff -r ae1531e00344 -r 7899867c7bf5 pkg/geoserver/boot.go --- a/pkg/geoserver/boot.go Mon Oct 08 14:52:37 2018 +0200 +++ b/pkg/geoserver/boot.go Mon Oct 08 14:53:17 2018 +0200 @@ -3,13 +3,13 @@ import ( "bytes" "encoding/json" + "encoding/xml" "fmt" + "io" "log" - "net" "net/http" "net/url" "strings" - "time" "gemma.intevation.de/gemma/pkg/config" "gemma.intevation.de/gemma/pkg/models" @@ -33,13 +33,27 @@ } } +func toStream(x interface{}) io.Reader { + var buf bytes.Buffer + + if err := json.NewEncoder(&buf).Encode(x); err != nil { + // Should not happen + log.Printf("bad JSON: %v\n", err) + } + return bytes.NewReader(buf.Bytes()) +} + func asJSON(req *http.Request) { req.Header.Set("Content-Type", "application/json") } +func asContentType(req *http.Request, contentType string) { + req.Header.Set("Content-Type", contentType) +} + func ensureWorkspace() error { var ( - url = config.GeoServerURL() + geoURL = config.GeoServerURL() user = config.GeoServerUser() password = config.GeoServerPassword() auth = basicAuth(user, password) @@ -48,7 +62,7 @@ // Probe workspace. req, err := http.NewRequest( http.MethodGet, - url+"/rest/workspaces/"+workspaceName+".json", + geoURL+"/rest/workspaces/"+workspaceName+".json", nil) if err != nil { return err @@ -72,7 +86,7 @@ req, err = http.NewRequest( http.MethodPost, - url+"/rest/workspaces", + geoURL+"/rest/workspaces", strings.NewReader(createJSON)) if err != nil { return err @@ -94,7 +108,7 @@ func ensureDataStore() error { var ( - url = config.GeoServerURL() + geoURL = config.GeoServerURL() user = config.GeoServerUser() password = config.GeoServerPassword() auth = basicAuth(user, password) @@ -103,7 +117,8 @@ // Probe datastore. req, err := http.NewRequest( http.MethodGet, - url+"/rest/workspaces/"+workspaceName+"/datastores/"+datastoreName+".json", + geoURL+"/rest/workspaces/"+workspaceName+ + "/datastores/"+datastoreName+".json", nil) if err != nil { return err @@ -146,16 +161,11 @@ }, }, } - var out bytes.Buffer - enc := json.NewEncoder(&out) - if err := enc.Encode(&ds); err != nil { - return err - } req, err = http.NewRequest( http.MethodPost, - url+"/rest/workspaces/"+workspaceName+"/datastores", - bytes.NewReader(out.Bytes())) + geoURL+"/rest/workspaces/"+workspaceName+"/datastores", + toStream(ds)) if err != nil { return err } @@ -177,7 +187,7 @@ func ensureFeatures() error { var ( - url = config.GeoServerURL() + geoURL = config.GeoServerURL() user = config.GeoServerUser() password = config.GeoServerPassword() auth = basicAuth(user, password) @@ -211,7 +221,7 @@ // Fetch all featuretypes. req, err := http.NewRequest( http.MethodGet, - url+"/rest/workspaces/"+workspaceName+ + geoURL+"/rest/workspaces/"+workspaceName+ "/datastores/"+datastoreName+ "/featuretypes.json", nil) @@ -255,18 +265,12 @@ }, } - var out bytes.Buffer - enc := json.NewEncoder(&out) - if err := enc.Encode(&ft); err != nil { - return err - } - req, err := http.NewRequest( http.MethodPost, - url+"/rest/workspaces/"+workspaceName+ + geoURL+"/rest/workspaces/"+workspaceName+ "/datastores/"+datastoreName+ "/featuretypes", - bytes.NewReader(out.Bytes())) + toStream(ft)) if err != nil { return err } @@ -287,7 +291,239 @@ return nil } -func prepareGeoServer() error { +func deleteWorkspace() error { + + // Should we delete our workspace first? + if !config.GeoServerClean() { + return nil + } + + log.Println("info: delete workspace " + workspaceName) + var ( + geoURL = config.GeoServerURL() + user = config.GeoServerUser() + password = config.GeoServerPassword() + auth = basicAuth(user, password) + ) + + req, err := http.NewRequest( + http.MethodDelete, + geoURL+"/rest/workspaces/"+workspaceName+"?recurse=true", + nil) + if err != nil { + return err + } + auth(req) + _, err = http.DefaultClient.Do(req) + return err +} + +type styles struct { + Styles struct { + Style []struct { + Name string `json:"name"` + } `json:"style"` + } `json:"styles"` +} + +func (s *styles) hasStyle(name string) bool { + for i := range s.Styles.Style { + if s.Styles.Style[i].Name == name { + return true + } + } + return false +} + +func (s *styles) load() error { + var ( + geoURL = config.GeoServerURL() + user = config.GeoServerUser() + password = config.GeoServerPassword() + auth = basicAuth(user, password) + ) + + req, err := http.NewRequest( + http.MethodGet, + geoURL+"/rest/workspaces/"+workspaceName+"/styles.json", + nil) + if err != nil { + return err + } + auth(req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // Fetch all styles + if err := json.NewDecoder(resp.Body).Decode(s); err != nil { + // XXX: Same quirk as with featuretypes. + } + return nil +} + +func updateStyle(entry *models.IntEntry, create bool) error { + + log.Printf("creating style %s\n", entry.Name) + + // Try to load the style data. + data, err := entry.LoadStyle() + if err != nil { + return err + } + + var ( + geoURL = config.GeoServerURL() + user = config.GeoServerUser() + password = config.GeoServerPassword() + auth = basicAuth(user, password) + ) + + styleURL := geoURL + "/rest/workspaces/" + workspaceName + + "/styles" + + // First create style + + type Style struct { + Name string `json:"name"` + Filename string `json:"filename"` + } + + var styleFilename = struct { + Style Style `json:"style"` + }{ + Style: Style{ + Name: entry.Name, + Filename: entry.Name + ".sld", + }, + } + + if create { + req, err := http.NewRequest( + http.MethodPost, + styleURL, + toStream(&styleFilename)) + + if err != nil { + return err + } + auth(req) + asJSON(req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusCreated { + return fmt.Errorf("Unable to create style %s (%s)", + entry.Name, + http.StatusText(resp.StatusCode)) + } + } + + // Second upload data + + req, err := http.NewRequest( + http.MethodPut, + styleURL+"/"+url.PathEscape(entry.Name), + strings.NewReader(data)) + if err != nil { + return err + } + auth(req) + if isSymbologyEncoding(data) { + asContentType(req, "application/vnd.ogc.se+xml") + } else { + asContentType(req, "application/vnd.ogc.sld+xml") + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("cannot upload style %s (%s)", + entry.Name, http.StatusText(resp.StatusCode)) + } + + // Third associate with layer + + if create { + req, err := http.NewRequest( + http.MethodPost, + geoURL+"/rest/layers/"+ + url.PathEscape(workspaceName+":"+entry.Name)+ + "/styles?default=true", + toStream(&styleFilename)) + if err != nil { + return err + } + auth(req) + asJSON(req) + + resp, err = http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusCreated { + return fmt.Errorf("cannot connect style %s with layer (%s)", + entry.Name, http.StatusText(resp.StatusCode)) + } + } + + return nil +} + +// isSymbologyEncoding tries to figure out if its plain SLD or SE. +func isSymbologyEncoding(data string) bool { + decoder := xml.NewDecoder(strings.NewReader(data)) + for { + tok, err := decoder.Token() + switch { + case tok == nil && err == io.EOF: + return false + case err != nil: + log.Printf("warn: invalid XML: %v\n", err) + return false + } + if t, ok := tok.(xml.StartElement); ok && + t.Name.Space == "http://www.opengis.net/se" { + return true + } + } +} + +func ensureStyles() error { + log.Println("info: creating styles") + + var stls styles + if err := stls.load(); err != nil { + return err + } + + entries := models.InternalServices.Filter( + models.IntAnd( + models.IntWMS, + models.IntWithStyle)) + + for i := range entries { + entry := &entries[i] + if stls.hasStyle(entry.Name) { + log.Printf("already has style for %s\n", entry.Name) + continue + } + if err := updateStyle(entry, true); err != nil { + return err + } + } + + return nil +} + +func PrepareGeoServer() error { if config.DBUser() == "" { log.Println("info: Need metamorphic db user to configure GeoServer") @@ -295,44 +531,46 @@ } if config.GeoServerURL() == "" { - log.Println("info: No tables to publish on GeoServer") + log.Println("info: No URL to GeoServer configured") return nil } - if err := ensureWorkspace(); err != nil { - return err + for _, ensure := range []func() error{ + deleteWorkspace, + ensureWorkspace, + ensureDataStore, + ensureFeatures, + ensureStyles, + } { + if err := ensure(); err != nil { + return err + } } - if err := ensureDataStore(); err != nil { - return err - } - - // TODO: Styles - - return ensureFeatures() + return nil } -func ConfigureBoot() { - log.Println("Configure GeoServer...") - const maxTries = 10 - const sleep = time.Second * 5 +func ReconfigureStyle(name string) { + Reconfigure(func() error { + var stls styles + if err := stls.load(); err != nil { + return err + } - for try := 1; try <= maxTries; try++ { - err := prepareGeoServer() - if err == nil { - break - } - if try < maxTries { - if uerr, ok := err.(*url.Error); ok { - if oerr, ok := uerr.Err.(*net.OpError); ok && oerr.Op == "dial" { - log.Printf("Failed attempt %d of %d to configure GeoServer. "+ - "Will try again in %s...\n", try, maxTries, sleep) - time.Sleep(sleep) - continue - } + entries := models.InternalServices.Filter( + models.IntAnd( + models.IntWMS, + models.IntWithStyle, + models.IntByName(name))) + + for i := range entries { + entry := &entries[i] + create := !stls.hasStyle(entry.Name) + if err := updateStyle(entry, create); err != nil { + return err } } - log.Printf("warn: configure GeoServer failed: %v\n", err) - break - } + + return nil + }) } diff -r ae1531e00344 -r 7899867c7bf5 pkg/geoserver/reconf.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/geoserver/reconf.go Mon Oct 08 14:53:17 2018 +0200 @@ -0,0 +1,67 @@ +package geoserver + +import ( + "container/list" + "log" + "net" + "net/url" + "sync" + "time" +) + +var ( + confQueue = list.New() + confQueueCond = sync.NewCond(new(sync.Mutex)) +) + +func init() { + go asyncConfigure() +} + +func asyncConfigure() { + for { + var fn func() error + confQueueCond.L.Lock() + for confQueue.Len() == 0 { + confQueueCond.Wait() + } + fn = confQueue.Remove(confQueue.Front()).(func() error) + confQueueCond.L.Unlock() + if err := reconfigure(fn); err != nil { + log.Printf("warn: configure GeoServer failed: %v\n", err) + } + } +} + +func reconfigure(fn func() error) error { + log.Println("Configure GeoServer...") + const ( + maxTries = 10 + sleep = time.Second * 5 + ) + var err error + for try := 1; try <= maxTries; try++ { + if err = fn(); err == nil { + break + } + if try < maxTries { + if uerr, ok := err.(*url.Error); ok { + if oerr, ok := uerr.Err.(*net.OpError); ok && oerr.Op == "dial" { + log.Printf("Failed attempt %d of %d to configure GeoServer. "+ + "Will try again in %s...\n", try, maxTries, sleep) + time.Sleep(sleep) + continue + } + } + } + break + } + return err +} + +func Reconfigure(fn func() error) { + confQueueCond.L.Lock() + defer confQueueCond.L.Unlock() + confQueue.PushBack(fn) + confQueueCond.Signal() +} diff -r ae1531e00344 -r 7899867c7bf5 pkg/models/intservices.go --- a/pkg/models/intservices.go Mon Oct 08 14:52:37 2018 +0200 +++ b/pkg/models/intservices.go Mon Oct 08 14:53:17 2018 +0200 @@ -4,6 +4,7 @@ "context" "database/sql" "log" + "net/http" "sync" "gemma.intevation.de/gemma/pkg/auth" @@ -11,10 +12,10 @@ ) type IntEntry struct { - Name string `json:"name"` - Style sql.NullString `json:"-"` // This should be done separately. - WMS bool `json:"wms"` - WFS bool `json:"wfs"` + Name string `json:"name"` + Style bool `json:"style"` + WMS bool `json:"wms"` + WFS bool `json:"wfs"` } type IntServices struct { @@ -22,11 +23,50 @@ mu sync.Mutex } -const selectPublishedServices = `SELECT relname, style, as_wms, as_wfs -FROM sys_admin.published_services JOIN pg_class ON name = oid ORDER by relname` +const ( + selectServicesSQL = ` +SELECT relname, style IS NOT NULL, as_wms, as_wfs +FROM sys_admin.published_services +JOIN pg_class ON name = oid ORDER by relname` + + selectStyleSQL = ` +SELECT style IS NOT NULL +FROM sys_admin.published_services +JOIN pg_class ON name = oid +WHERE relname = $1` + + updateStyleSQL = ` +UPDATE sys_admin.published_services +SET style = $1::bytea +WHERE name IN (SELECT oid FROM pg_class WHERE relname = $2)` +) var InternalServices = &IntServices{} +func (e *IntEntry) LoadStyle() (string, error) { + var style string + err := auth.RunAs("sys_admin", context.Background(), + func(conn *sql.Conn) error { + return conn.QueryRowContext( + context.Background(), + selectStyleSQL, + e.Name).Scan(&style) + }) + return style, err +} + +func UpdateInternalStyle(req *http.Request, name, style string) error { + return auth.RunAsSessionUser(req, func(conn *sql.Conn) error { + _, err := conn.ExecContext( + req.Context(), updateStyleSQL, + style, name) + if err == nil { + InternalServices.Invalidate() + } + return err + }) +} + func (ps *IntServices) Find(name string) (string, bool) { ps.mu.Lock() defer ps.mu.Unlock() @@ -68,7 +108,7 @@ return auth.RunAs("sys_admin", context.Background(), func(conn *sql.Conn) error { rows, err := conn.QueryContext( - context.Background(), selectPublishedServices) + context.Background(), selectServicesSQL) if err != nil { return err } @@ -93,9 +133,25 @@ ps.mu.Unlock() } -func InternalAll(IntEntry) bool { return true } -func IntWMS(entry IntEntry) bool { return entry.WMS } -func IntWFS(entry IntEntry) bool { return entry.WFS } +func InternalAll(IntEntry) bool { return true } +func IntWMS(entry IntEntry) bool { return entry.WMS } +func IntWFS(entry IntEntry) bool { return entry.WFS } +func IntWithStyle(entry IntEntry) bool { return entry.Style } + +func IntByName(name string) func(IntEntry) bool { + return func(entry IntEntry) bool { return entry.Name == name } +} + +func IntAnd(accept ...func(IntEntry) bool) func(IntEntry) bool { + return func(entry IntEntry) bool { + for _, a := range accept { + if !a(entry) { + return false + } + } + return true + } +} func (ps *IntServices) Filter(accept func(IntEntry) bool) []IntEntry { ps.mu.Lock() diff -r ae1531e00344 -r 7899867c7bf5 schema/auth.sql --- a/schema/auth.sql Mon Oct 08 14:52:37 2018 +0200 +++ b/schema/auth.sql Mon Oct 08 14:53:17 2018 +0200 @@ -33,6 +33,7 @@ GRANT SELECT ON ALL TABLES IN SCHEMA sys_admin TO sys_admin; GRANT UPDATE ON sys_admin.system_config TO sys_admin; GRANT UPDATE ON systemconf.feature_colours TO sys_admin; +GRANT UPDATE ON sys_admin.published_services TO sys_admin; GRANT INSERT, DELETE ON sys_admin.password_reset_requests TO sys_admin; GRANT INSERT, DELETE, UPDATE ON waterway.sounding_results_contour_lines TO sys_admin; diff -r ae1531e00344 -r 7899867c7bf5 schema/gemma.sql