Mercurial > gemma

changeset 3580:d38b20ccb6f9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
GM import: Be more graceful when accepting measure/issue dates.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Mon, 03 Jun 2019 17:55:49 +0200
parents 571801483a70
children 9114097964d7
files pkg/imports/gm.go pkg/soap/nts/service.go pkg/soap/soap.go
diffstat 3 files changed, 38 insertions(+), 27 deletions(-) [+]
line wrap: on
line diff
--- a/pkg/imports/gm.go	Mon Jun 03 17:21:18 2019 +0200
+++ b/pkg/imports/gm.go	Mon Jun 03 17:55:49 2019 +0200
@@ -343,7 +343,6 @@
 
 	var gids []string
 	for _, msg := range result {
-		var dummy int
 		for _, wrm := range msg.Wrm {
 			curr := string(*wrm.Geo_object.Id)
 			currIsrs, err := models.IsrsFromString(curr)
@@ -388,6 +387,7 @@
 					continue
 				}
 
+				var dummy int
 				if measure.Predicted {
 					var confInterval pgtype.Numrange
 					if measure.Value_min != nil && measure.Value_max != nil {
@@ -409,15 +409,15 @@
 						currIsrs.FairwaySection,
 						currIsrs.Orc,
 						currIsrs.Hectometre,
-						measure.Measuredate,
+						measure.Measuredate.Time,
 						msg.Identification.From,
 						msg.Identification.Language_code,
 						msg.Identification.Country_code,
-						msg.Identification.Date_issue,
+						msg.Identification.Date_issue.Time,
 						referenceCode,
 						measure.Value,
 						&confInterval,
-						msg.Identification.Date_issue,
+						msg.Identification.Date_issue.Time,
 						msg.Identification.Originator,
 					).Scan(&dummy)
 					switch {
@@ -441,14 +441,14 @@
 						currIsrs.FairwaySection,
 						currIsrs.Orc,
 						currIsrs.Hectometre,
-						measure.Measuredate,
+						measure.Measuredate.Time,
 						msg.Identification.From,
 						msg.Identification.Language_code,
 						msg.Identification.Country_code,
-						msg.Identification.Date_issue,
+						msg.Identification.Date_issue.Time,
 						referenceCode,
 						measure.Value,
-						msg.Identification.Date_issue,
+						msg.Identification.Date_issue.Time,
 						msg.Identification.Originator,
 					).Scan(&dummy)
 					switch {
--- a/pkg/soap/nts/service.go	Mon Jun 03 17:21:18 2019 +0200
+++ b/pkg/soap/nts/service.go	Mon Jun 03 17:55:49 2019 +0200
@@ -18,12 +18,29 @@
 	"encoding/xml"
 	"time"
 
+	"gemma.intevation.de/gemma/pkg/misc"
 	"gemma.intevation.de/gemma/pkg/soap"
 )
 
-// against "unused imports"
-var _ time.Time
-var _ xml.Name
+var guessDateTime = misc.TimeGuesser([]string{
+	"2006-01-02T15:04:05",
+	"2006-01-02T15:04:05-07:00",
+}).Guess
+
+type DateTime struct{ time.Time }
+
+func (dt *DateTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	var value string
+	if err := d.DecodeElement(&value, &start); err != nil {
+		return err
+	}
+	t, err := guessDateTime(value)
+	if err != nil {
+		return err
+	}
+	*dt = DateTime{t}
+	return nil
+}
 
 type Message_type_type string
 
@@ -124,7 +141,7 @@
 }
 
 type Paging_result_type struct {
-	XMLName xml.Name `xml:"http://www.ris.eu/nts/4.0.4.0 paging_result"`
+	XMLName xml.Name `xml:"http://www.ris.eu/nts.ms/2.0.4.0 paging_result"`
 
 	Offset *NonNegativeInteger `xml:"offset,omitempty"`
 
@@ -1172,7 +1189,7 @@
 	District string `xml:"district,omitempty"`
 
 	// Date and time of publication including time zone
-	Date_issue time.Time `xml:"date_issue,omitempty"`
+	Date_issue DateTime `xml:"date_issue,omitempty"`
 }
 
 type Ftm_type struct {
@@ -1291,7 +1308,7 @@
 	Regime_code *Regime_code_enum `xml:"regime_code,omitempty"`
 
 	// Date and Time of measurement or predicted value including time zone
-	Measuredate time.Time `xml:"measuredate,omitempty"`
+	Measuredate DateTime `xml:"measuredate,omitempty"`
 
 	// Difference with comparative value
 	Difference *Difference_type `xml:"difference,omitempty"`
@@ -1330,7 +1347,7 @@
 	XMLName xml.Name `xml:"http://www.ris.eu/nts/4.0.4.0 ice_condition_type"`
 
 	// Date and Time of measurement or prediction including time zone
-	Measuredate time.Time `xml:"measuredate,omitempty"`
+	Measuredate DateTime `xml:"measuredate,omitempty"`
 
 	// Condition code
 	Ice_condition_code *Ice_condition_code_enum `xml:"ice_condition_code,omitempty"`
@@ -1375,7 +1392,7 @@
 	XMLName xml.Name `xml:"http://www.ris.eu/nts/4.0.4.0 weather_report_type"`
 
 	// Date and time of measurement or predicted value including timezone
-	Measuredate time.Time `xml:"measuredate,omitempty"`
+	Measuredate DateTime `xml:"measuredate,omitempty"`
 
 	// Forecast (true or 1) OR Actual report (false or 0)
 	Forecast bool `xml:"forecast,omitempty"`
--- a/pkg/soap/soap.go	Mon Jun 03 17:21:18 2019 +0200
+++ b/pkg/soap/soap.go	Mon Jun 03 17:55:49 2019 +0200
@@ -30,12 +30,6 @@
 	"gemma.intevation.de/gemma/pkg/config"
 )
 
-const timeout = time.Duration(30 * time.Second)
-
-func dialTimeout(network, addr string) (net.Conn, error) {
-	return net.DialTimeout(network, addr, timeout)
-}
-
 type SOAPEnvelope struct {
 	XMLName xml.Name `xml:"http://schemas.xmlsoap.org/soap/envelope/ Envelope"`
 	Header  *SOAPHeader
@@ -253,8 +247,6 @@
 		return err
 	}
 
-	//log.Println(buffer.String())
-
 	req, err := http.NewRequest("POST", s.url, buffer)
 	if err != nil {
 		return err
@@ -269,15 +261,17 @@
 	req.Header.Set("User-Agent", "gowsdl/0.1")
 	req.Close = true
 
+	timeout := config.SOAPTimeout()
+
 	tr := &http.Transport{
 		TLSClientConfig: s.tlsCfg,
-		Dial:            dialTimeout,
+		Dial: func(network, addr string) (net.Conn, error) {
+			return net.DialTimeout(network, addr, timeout)
+		},
 	}
 
 	client := &http.Client{Transport: tr}
 
-	timeout := config.SOAPTimeout()
-
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 
 	var once sync.Once
@@ -306,7 +300,7 @@
 		return err
 	}
 	if len(rawbody) == 0 {
-		log.Println("empty response")
+		log.Println("warn: empty response")
 		return nil
 	}