hancock

Check-in [99044d8f59]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:hancock: call new server interface previously: seperate index and testimony on IPFS now: hancockd server acts as key/value store
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 99044d8f593b41e9270a582d2cc5b46f31c572e528556fbaf55add2f7411cd74
User & Date: dnc 2019-11-03 12:01:55
Context
2019-12-11
20:19
model: helper to encode testimony key check-in: 812804e091 user: dnc tags: trunk
2019-11-03
12:01
hancock: call new server interface previously: seperate index and testimony on IPFS now: hancockd server acts as key/value store check-in: 99044d8f59 user: dnc tags: trunk
11:59
model: rename manifest "Content" field (was "CID") check-in: da5f7ebb26 user: dnc tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to cmd/hancock/publish.go.

35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
..
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
...
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
...
203
204
205
206
207
208
209
210
211





212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227






228
229
230
231
232
233
234
235
236
237







238
239
240
241

242
243
244
245

246
247
248
249
250
251
252
253
254
	"src.d10.dev/command"
	"src.d10.dev/hancock/model"
)

var (
	// Flags that may be used by both publish and testimony operations.

	// single IPFS gateway
	ipfsFlag *string

	// one or more indexes
	indexFlag command.StringSet
)

func validateIPFSFlag() error {
	if ipfsFlag == nil || *ipfsFlag == "" {
		return nil
	}
	u, err := url.Parse(*ipfsFlag)
	if err != nil {
		command.Error(errors.Wrapf(err, "bad IPFS gateway URL (%s)", *ipfsFlag))
		command.Exit()
	}
	switch u.Scheme {
	case "http", "https":
		// supported
	default:
		command.Errorf("bad IPFS gateway (%q): scheme must be http(s)://", *ipfsFlag)
		command.Exit()
	}
	return err
}

func validateIndexFlag(defaultIndex string) error {

	if len(indexFlag) == 0 && defaultIndex != "" {
		indexFlag = append(indexFlag, defaultIndex)
	}
	for _, val := range indexFlag {
		if val == "" {
			continue
		}
		u, err := url.Parse(val)
		if err != nil {
			command.Error(errors.Wrapf(err, "bad index URL (%s)", val))
			command.Exit()
................................................................................
	return nil
}

func init() {
	command.RegisterOperation(command.Operation{
		Handler:     publishMain,
		Name:        "publish",
		Syntax:      "publish [-index=<url>] [-ipfs=<url>] [in=<directory] [testimony ...]",
		Description: "Publish signed manifest to an index. Expects signed activity on stdin, as encoded by `hancock-sign`.",
	})
}

func publishMain() error {
	cfg, err := command.Config()
	command.Check(err)

	// defaults
	cfgTop := cfg.Section("")
	cfgOp := cfg.Section("publish")

	defaultIndex := cfgOp.Key("index").MustString(cfgTop.Key("index").MustString("https://hancock.beyondcentral.com"))
	defaultIPFS := cfgOp.Key("ipfs").MustString(cfgTop.Key("ipfs").MustString("http://localhost:8080"))
	defaultDirectory := cfgOp.Key("testimony").MustString(cfgTop.Key("testimony").String())

	// args
	command.OperationFlagSet.Var(&indexFlag, "index", fmt.Sprintf("URL of testimony index (default %s)", defaultIndex))
	ipfsFlag = command.OperationFlagSet.String("ipfs", defaultIPFS, "URL of IPFS gateway")
	inFlag := command.OperationFlagSet.String("in", defaultDirectory, "directory containing testimony files")

	// parse flags
	err = command.OperationFlagSet.Parse(command.Args()[1:])
	if err != nil {
		return err
	}

	// validate flags
	err = validateIndexFlag(defaultIndex)
	if err != nil {
		return err
	}
	if len(indexFlag) == 0 {
		return errors.New("Index URL required. Use flag `-index <URL>` or see configuration file.")
	}

	if len(command.OperationFlagSet.Args()) == 0 && *inFlag == "" {
		return errors.New("Testimony required.  Use `-in <directory>` or specify testimony files.")
	}
	if *inFlag != "" {
		inStat, err := os.Stat(*inFlag)
................................................................................
			}
			err = testimony.FileManifest.Check()
			if err != nil {
				return err
			}

			// validate testimony here?
			indexed := false
			for _, idxURL := range indexFlag {
				err = publishTestimonyToIndex(idxURL, testimony)
				if err != nil {
					command.Error(err)
				} else {
					indexed = true
				}
			}
			if indexed {
				count++
			}
			return err
		})
		command.V(1).Infof("Published %d items from %q.", count, fileOrDir)
		if err != nil {
			command.Error(err)
................................................................................

func publishTestimonyToIPFS(ipfsURL string, ft fileTestimony) error {
	err := ipfsAdd(ipfsURL, ft.Bytes(), ft.FileName())
	return err
}

// publish testimony either to local filesystem or via POST to online index.
func publishTestimonyToIndex(idxURL string, ft fileTestimony) error {
	u, err := url.Parse(idxURL)





	if err != nil {
		return err
	}

	switch u.Scheme {
	case "http", "https":
		// post to URL
		urlPath := idxURL + "/testimony"
		resp, err := http.Post(urlPath, "application/json", bytes.NewBuffer(ft.Bytes()))
		if err != nil {
			return err
		}
		defer resp.Body.Close()
		switch resp.StatusCode {
		case http.StatusOK, http.StatusCreated, http.StatusAccepted:
			// success






		default:
			command.V(1).Infof("Failed to post testimony: %s", string(ft.Bytes()))
			b, _ := ioutil.ReadAll(resp.Body)
			return fmt.Errorf("failed to publish testimony (%q) to index (%s): %s %q", ft.Path, idxURL, resp.Status, string(b))
		}

	case "file":
		// save to directory
		indexValue := model.NewSha256CID(ft.Bytes())
		indexKey, err := model.IndexKey(ft.CID, ft.Public)







		path := filepath.Join(u.Path, filepath.Join(indexKey...))
		err = os.MkdirAll(filepath.Dir(path), os.ModePerm)
		if err != nil {
			return errors.Wrapf(err, "failed to save index file (%q)", path)

		}
		err = ioutil.WriteFile(path, []byte(indexValue.String()), 0644)
		if err != nil {
			return errors.Wrapf(err, "failed to save index file (%q)", path)

		} else {
			command.V(2).Infof("wrote testimony (%q) index file (%q)", ft.Path, path)
		}

	default:
		return fmt.Errorf("unexpected index URL scheme (%q)", u.Scheme)
	}
	return nil
}







<
<
<
<
|


|
<
<
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
|

|







 







|












|
<



|
<









|



|
|







 







|
|
|



|


|







 







|
|
>
>
>
>
>







|








>
>
>
>
>
>



|




<
|
>
>
>
>
>
>
>
|


<
>

|

<
>

|







35
36
37
38
39
40
41




42
43
44
45


46

















47
48
49
50
51
52
53
54
55
56
57
..
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

88
89
90
91

92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
...
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
...
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221

222
223
224
225
226
227
228
229
230
231
232

233
234
235
236

237
238
239
240
241
242
243
244
245
246
	"src.d10.dev/command"
	"src.d10.dev/hancock/model"
)

var (
	// Flags that may be used by both publish and testimony operations.





	storeFlag command.StringSet
)

func validateStoreFlag(def string) error {




















	if len(storeFlag) == 0 && def != "" {
		storeFlag = append(storeFlag, def)
	}
	for _, val := range storeFlag {
		if val == "" {
			continue
		}
		u, err := url.Parse(val)
		if err != nil {
			command.Error(errors.Wrapf(err, "bad index URL (%s)", val))
			command.Exit()
................................................................................
	return nil
}

func init() {
	command.RegisterOperation(command.Operation{
		Handler:     publishMain,
		Name:        "publish",
		Syntax:      "publish [-store=<url>] [in=<directory] [testimony ...]",
		Description: "Publish signed manifest to an index. Expects signed activity on stdin, as encoded by `hancock-sign`.",
	})
}

func publishMain() error {
	cfg, err := command.Config()
	command.Check(err)

	// defaults
	cfgTop := cfg.Section("")
	cfgOp := cfg.Section("publish")

	defaultStore := cfgOp.Key("store").MustString(cfgTop.Key("store").MustString("https://hancock.beyondcentral.com"))

	defaultDirectory := cfgOp.Key("testimony").MustString(cfgTop.Key("testimony").String())

	// args
	command.OperationFlagSet.Var(&storeFlag, "store", fmt.Sprintf("URL of testimony store (default %q)", defaultStore))

	inFlag := command.OperationFlagSet.String("in", defaultDirectory, "directory containing testimony files")

	// parse flags
	err = command.OperationFlagSet.Parse(command.Args()[1:])
	if err != nil {
		return err
	}

	// validate flags
	err = validateStoreFlag(defaultStore)
	if err != nil {
		return err
	}
	if len(storeFlag) == 0 {
		return errors.New("Storage URL required. Use flag `-store <URL>` or see configuration file.")
	}

	if len(command.OperationFlagSet.Args()) == 0 && *inFlag == "" {
		return errors.New("Testimony required.  Use `-in <directory>` or specify testimony files.")
	}
	if *inFlag != "" {
		inStat, err := os.Stat(*inFlag)
................................................................................
			}
			err = testimony.FileManifest.Check()
			if err != nil {
				return err
			}

			// validate testimony here?
			stored := false
			for _, storeURL := range storeFlag {
				err = publishTestimonyToStore(storeURL, testimony)
				if err != nil {
					command.Error(err)
				} else {
					stored = true
				}
			}
			if stored {
				count++
			}
			return err
		})
		command.V(1).Infof("Published %d items from %q.", count, fileOrDir)
		if err != nil {
			command.Error(err)
................................................................................

func publishTestimonyToIPFS(ipfsURL string, ft fileTestimony) error {
	err := ipfsAdd(ipfsURL, ft.Bytes(), ft.FileName())
	return err
}

// publish testimony either to local filesystem or via POST to online index.
func publishTestimonyToStore(storeURL string, ft fileTestimony) error {
	u, err := url.Parse(storeURL)
	if err != nil {
		return err
	}

	err = ft.Check()
	if err != nil {
		return err
	}

	switch u.Scheme {
	case "http", "https":
		// post to URL
		urlPath := storeURL + "/testimony"
		resp, err := http.Post(urlPath, "application/json", bytes.NewBuffer(ft.Bytes()))
		if err != nil {
			return err
		}
		defer resp.Body.Close()
		switch resp.StatusCode {
		case http.StatusOK, http.StatusCreated, http.StatusAccepted:
			// success
			if command.V(2) {
				command.Infof("posted testimony to %q", urlPath)
				b, _ := ioutil.ReadAll(resp.Body)
				command.Info(resp.StatusCode)
				command.Info(string(b))
			}
		default:
			command.V(1).Infof("Failed to post testimony: %s", string(ft.Bytes()))
			b, _ := ioutil.ReadAll(resp.Body)
			return fmt.Errorf("failed to publish testimony (%q) to index (%s): %s %q", ft.Path, storeURL, resp.Status, string(b))
		}

	case "file":
		// save to directory

		key, err := model.IndexKey(ft.Testimony.Content, ft.Authority)
		if err != nil {
			return err
		}

		// filepath.Join() turns the key values into directories.  If we
		// used flat file names, we might exceed filesystem limits for
		// number of files.
		path := filepath.Join(u.Path, filepath.Join(key...))
		err = os.MkdirAll(filepath.Dir(path), os.ModePerm)
		if err != nil {

			return fmt.Errorf("failed to store testimony file (%q): %w", path, err)
		}
		err = ioutil.WriteFile(path, ft.Bytes(), 0644)
		if err != nil {

			return fmt.Errorf("failed to store testimony file (%q): %w", path, err)
		} else {
			command.V(2).Infof("wrote testimony (%q) to file (%q)", ft.Path, path)
		}

	default:
		return fmt.Errorf("unexpected index URL scheme (%q)", u.Scheme)
	}
	return nil
}

Changes to cmd/hancock/testimony.go.

23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69




70
71
72
73
74
75
76
77
78
79
80
81
82








83
84
85
86
87
88
89
90
91

92
93
94
95
96
97
98

99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
...
154
155
156
157
158
159
160
161

162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190

191
192
193

194
195
196
197
198
199
200
// to an index.
package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"github.com/pkg/errors"
	"golang.org/x/crypto/ssh"

	"src.d10.dev/command"
	"src.d10.dev/hancock/model"
)

func init() {
	command.RegisterOperation(command.Operation{
		Handler:     testimonyMain,
		Name:        "testimony",
		Syntax:      "testimony [-out <directory>] [-index=<url>] [-ipfs=<url>]",
		Description: "Publish signed manifest to an index. Expects signed activity on stdin, as encoded by `hancock-sign`.",
	})
}

// fileTestimony bundles manifest and signature, including (redundant)
// encoding of manifest to ensure its CID does not change.
type fileTestimony struct {
	model.Testimony
	model.FileManifest `json:"manifest"`

	// calculate these once, use when publishing to multiple indexes and IPFS.
	encoded  []byte
	filename string
}

func (this *fileTestimony) FileName() string {
	if this.filename == "" {
		pub, err := this.PublicKey()
		if err != nil {
			log.Panic(err)
		}
		this.filename = fmt.Sprintf("%s-%x.testimony", this.CID, pub.Marshal()) // convention: CID of source, pubkey of sig.
	}




	return this.filename
}

func (this *fileTestimony) Bytes() []byte {
	if this.encoded == nil {
		var err error
		this.encoded, err = model.Encode(this)
		if err != nil {
			log.Panic(err)
		}
	}
	return this.encoded
}









func testimonyMain() error {
	cfg, err := command.Config()
	command.Check(err)

	// defaults
	cfgTop := cfg.Section("")
	cfgTestimony := cfg.Section("testimony")
	defaultDirectory := cfgTestimony.Key("directory").MustString(cfgTop.Key("testimony").String())

	defaultIndex := cfgTestimony.Key("index").MustString(cfgTop.Key("index").String())
	defaultIPFS := cfgTestimony.Key("ipfs").MustString(cfgTop.Key("ipfs").String())

	// define args
	outFlag := command.OperationFlagSet.String("out", defaultDirectory, "directory where testimony files are written")
	command.OperationFlagSet.Var(&indexFlag, "index", fmt.Sprintf("URL of testimony index (default %s)", defaultIndex))
	ipfsFlag = command.OperationFlagSet.String("ipfs", defaultIPFS, "add testimony via IPFS gateway")

	noopFlag := command.OperationFlagSet.Bool("n", false, "do not save or publish")

	// parse args
	err = command.OperationFlagSet.Parse(command.Args()[1:])
	if err != nil {
		return err
	}

	// validate args
	if *outFlag == "" {
		return errors.New("Output directory required.  Use `-out <directory>`")
	}
	outStat, err := os.Stat(*outFlag)
	if os.IsNotExist(err) {
		return fmt.Errorf("Output directory does not exist (%q).", *outFlag)
	}
	if !outStat.IsDir() {
		return fmt.Errorf("Not a directory (%q).", *outFlag)
	}

	err = validateIndexFlag(defaultIndex)
	if err != nil {
		return err
	}


	err = validateIPFSFlag()
	if err != nil {
		return err
	}

	// decode input
	dec := json.NewDecoder(os.Stdin)

	// first input, the public key (corresponding to signing private key)
	var publicEncoded string
	err = dec.Decode(&publicEncoded)
	command.Check(err)

	// ensure key parses
	public, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicEncoded)) // NOT ssh.ParsePublicKey()
	command.Check(err)
	_ = public // may be used in filename

	// following the public key, we read pairs; each pair a manifest followed by testimony
	for {
		var rawMan json.RawMessage
		err = dec.Decode(&rawMan)
................................................................................
		command.Check(man.Check())

		var sig ssh.Signature
		err = dec.Decode(&sig)
		command.Check(err)

		testimony := model.Testimony{
			Public:    publicEncoded,

			Encoded:   rawMan,
			Signature: sig,
		}

		if !*noopFlag {
			ft := fileTestimony{Testimony: testimony, FileManifest: man} // testimony with decoded manifest
			if *outFlag != "" {
				filepath := filepath.Join(*outFlag, ft.FileName())

				err = ioutil.WriteFile(filepath, ft.Bytes(), 0644)
				command.Check(err)
				command.V(2).Infof("%q wrote %s", man.Path, filepath)
			}

			if *ipfsFlag != "" {
				err = publishTestimonyToIPFS(*ipfsFlag, ft)
				if err != nil {
					command.Error(errors.Wrapf(err, "%q IPFS add failed (%s)", man.Path, *ipfsFlag))
				}
			}

			for _, idxURL := range indexFlag {
				if idxURL == "" {
					continue // explicit -index="" on command line
				}
				err = publishTestimonyToIndex(idxURL, ft)
				if err != nil {
					command.Errorf("%q failed to publish testimony to index (%s): %s", man.Path, idxURL, err)
				} else {

					command.V(2).Infof("%q testimony posted to index (%s)", man.Path, idxURL)
				}
			}

			command.V(1).Infof("%q testimony published", man.Path)
		}

	}

	return nil
}







<


<












|
|









|










|

>
>
>
>













>
>
>
>
>
>
>
>








<
>
|
<


<
<
<
>
|








<
<
<
<
<
<
<
<
<
<
<
|



<
>
|
<
<






|
|



|







 







|
>






|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
|


|

|

>
|


>
|
|
|




23
24
25
26
27
28
29

30
31

32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

101
102

103
104



105
106
107
108
109
110
111
112
113
114











115
116
117
118

119
120


121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
...
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163














164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
// to an index.
package main

import (
	"encoding/json"
	"fmt"
	"io"

	"log"
	"os"


	"github.com/pkg/errors"
	"golang.org/x/crypto/ssh"

	"src.d10.dev/command"
	"src.d10.dev/hancock/model"
)

func init() {
	command.RegisterOperation(command.Operation{
		Handler:     testimonyMain,
		Name:        "testimony",
		Syntax:      "testimony [-store=<url>]",
		Description: "Publish signed manifest to a hancockd server or local filesystem. Expects signed activity on stdin, as encoded by `hancock-sign`.",
	})
}

// fileTestimony bundles manifest and signature, including (redundant)
// encoding of manifest to ensure its CID does not change.
type fileTestimony struct {
	model.Testimony
	model.FileManifest `json:"manifest"`

	// calculate these once, use when publishing to multiple stores.
	encoded  []byte
	filename string
}

func (this *fileTestimony) FileName() string {
	if this.filename == "" {
		pub, err := this.PublicKey()
		if err != nil {
			log.Panic(err)
		}
		this.filename = fmt.Sprintf("%s-%x.testimony", this.Testimony.Content, pub.Marshal()) // convention: CID of source, pubkey of sig.
	}

	// still used?
	log.Panicf("fileTestimony.FileName() is still used: %q", this.filename)

	return this.filename
}

func (this *fileTestimony) Bytes() []byte {
	if this.encoded == nil {
		var err error
		this.encoded, err = model.Encode(this)
		if err != nil {
			log.Panic(err)
		}
	}
	return this.encoded
}

func (this *fileTestimony) Check() error {
	err := this.Testimony.Check()
	if err != nil {
		return err
	}
	return this.FileManifest.Check()
}

func testimonyMain() error {
	cfg, err := command.Config()
	command.Check(err)

	// defaults
	cfgTop := cfg.Section("")
	cfgTestimony := cfg.Section("testimony")


	defaultStore := cfgTestimony.Key("store").MustString(cfgTop.Key("store").String())


	// define args



	command.OperationFlagSet.Var(&storeFlag, "store", fmt.Sprintf("URL where testimony is saved/published (default %q)", defaultStore))
	noopFlag := command.OperationFlagSet.Bool("n", false, "do not save/publish")

	// parse args
	err = command.OperationFlagSet.Parse(command.Args()[1:])
	if err != nil {
		return err
	}

	// validate args











	err = validateStoreFlag(defaultStore)
	if err != nil {
		return err
	}

	if len(storeFlag) == 0 {
		return errors.New("Storage URL required. Use flag `-store <URL>` or see configuration file.")


	}

	// decode input
	dec := json.NewDecoder(os.Stdin)

	// first input, the public key (corresponding to signing private key)
	var authorityEncoded model.Authority
	err = dec.Decode(&authorityEncoded)
	command.Check(err)

	// ensure key parses
	public, _, _, _, err := ssh.ParseAuthorizedKey([]byte(authorityEncoded)) // NOT ssh.ParsePublicKey()
	command.Check(err)
	_ = public // may be used in filename

	// following the public key, we read pairs; each pair a manifest followed by testimony
	for {
		var rawMan json.RawMessage
		err = dec.Decode(&rawMan)
................................................................................
		command.Check(man.Check())

		var sig ssh.Signature
		err = dec.Decode(&sig)
		command.Check(err)

		testimony := model.Testimony{
			Authority: authorityEncoded,
			Content:   man.Content,
			Encoded:   rawMan,
			Signature: sig,
		}

		if !*noopFlag {
			ft := fileTestimony{Testimony: testimony, FileManifest: man} // testimony with decoded manifest
			stored := false














			for _, storeURL := range storeFlag {
				if storeURL == "" {
					continue // explicit -index="" on command line
				}
				err = publishTestimonyToStore(storeURL, ft)
				if err != nil {
					command.Errorf("%q failed to publish testimony to index (%s): %s", man.Path, storeURL, err)
				} else {
					stored = true
					command.V(2).Infof("%q testimony stored (%s)", man.Path, storeURL)
				}
			}
			if stored {
				command.V(1).Infof("%q testimony published", man.Path)
			}
		}
	}

	return nil
}