Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | hancock: call new server interface previously: seperate index and testimony on IPFS now: hancockd server acts as key/value store |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA3-256: |
99044d8f593b41e9270a582d2cc5b46f |
User & Date: | dnc 2019-11-03 12:01:55 |
Context
2019-12-11
| ||
20:19 | model: helper to encode testimony key check-in: 812804e091 user: dnc tags: trunk | |
2019-11-03
| ||
12:01 | hancock: call new server interface previously: seperate index and testimony on IPFS now: hancockd server acts as key/value store check-in: 99044d8f59 user: dnc tags: trunk | |
11:59 | model: rename manifest "Content" field (was "CID") check-in: da5f7ebb26 user: dnc tags: trunk | |
Changes
Changes to cmd/hancock/publish.go.
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 .. 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 ... 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 ... 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 |
"src.d10.dev/command" "src.d10.dev/hancock/model" ) var ( // Flags that may be used by both publish and testimony operations. // single IPFS gateway ipfsFlag *string // one or more indexes indexFlag command.StringSet ) func validateIPFSFlag() error { if ipfsFlag == nil || *ipfsFlag == "" { return nil } u, err := url.Parse(*ipfsFlag) if err != nil { command.Error(errors.Wrapf(err, "bad IPFS gateway URL (%s)", *ipfsFlag)) command.Exit() } switch u.Scheme { case "http", "https": // supported default: command.Errorf("bad IPFS gateway (%q): scheme must be http(s)://", *ipfsFlag) command.Exit() } return err } func validateIndexFlag(defaultIndex string) error { if len(indexFlag) == 0 && defaultIndex != "" { indexFlag = append(indexFlag, defaultIndex) } for _, val := range indexFlag { if val == "" { continue } u, err := url.Parse(val) if err != nil { command.Error(errors.Wrapf(err, "bad index URL (%s)", val)) command.Exit() ................................................................................ return nil } func init() { command.RegisterOperation(command.Operation{ Handler: publishMain, Name: "publish", Syntax: "publish [-index=<url>] [-ipfs=<url>] [in=<directory] [testimony ...]", Description: "Publish signed manifest to an index. Expects signed activity on stdin, as encoded by `hancock-sign`.", }) } func publishMain() error { cfg, err := command.Config() command.Check(err) // defaults cfgTop := cfg.Section("") cfgOp := cfg.Section("publish") defaultIndex := cfgOp.Key("index").MustString(cfgTop.Key("index").MustString("https://hancock.beyondcentral.com")) defaultIPFS := cfgOp.Key("ipfs").MustString(cfgTop.Key("ipfs").MustString("http://localhost:8080")) defaultDirectory := cfgOp.Key("testimony").MustString(cfgTop.Key("testimony").String()) // args command.OperationFlagSet.Var(&indexFlag, "index", fmt.Sprintf("URL of testimony index (default %s)", defaultIndex)) ipfsFlag = command.OperationFlagSet.String("ipfs", defaultIPFS, "URL of IPFS gateway") inFlag := command.OperationFlagSet.String("in", defaultDirectory, "directory containing testimony files") // parse flags err = command.OperationFlagSet.Parse(command.Args()[1:]) if err != nil { return err } // validate flags err = validateIndexFlag(defaultIndex) if err != nil { return err } if len(indexFlag) == 0 { return errors.New("Index URL required. Use flag `-index <URL>` or see configuration file.") } if len(command.OperationFlagSet.Args()) == 0 && *inFlag == "" { return errors.New("Testimony required. Use `-in <directory>` or specify testimony files.") } if *inFlag != "" { inStat, err := os.Stat(*inFlag) ................................................................................ } err = testimony.FileManifest.Check() if err != nil { return err } // validate testimony here? indexed := false for _, idxURL := range indexFlag { err = publishTestimonyToIndex(idxURL, testimony) if err != nil { command.Error(err) } else { indexed = true } } if indexed { count++ } return err }) command.V(1).Infof("Published %d items from %q.", count, fileOrDir) if err != nil { command.Error(err) ................................................................................ func publishTestimonyToIPFS(ipfsURL string, ft fileTestimony) error { err := ipfsAdd(ipfsURL, ft.Bytes(), ft.FileName()) return err } // publish testimony either to local filesystem or via POST to online index. func publishTestimonyToIndex(idxURL string, ft fileTestimony) error { u, err := url.Parse(idxURL) if err != nil { return err } switch u.Scheme { case "http", "https": // post to URL urlPath := idxURL + "/testimony" resp, err := http.Post(urlPath, "application/json", bytes.NewBuffer(ft.Bytes())) if err != nil { return err } defer resp.Body.Close() switch resp.StatusCode { case http.StatusOK, http.StatusCreated, http.StatusAccepted: // success default: command.V(1).Infof("Failed to post testimony: %s", string(ft.Bytes())) b, _ := ioutil.ReadAll(resp.Body) return fmt.Errorf("failed to publish testimony (%q) to index (%s): %s %q", ft.Path, idxURL, resp.Status, string(b)) } case "file": // save to directory indexValue := model.NewSha256CID(ft.Bytes()) indexKey, err := model.IndexKey(ft.CID, ft.Public) path := filepath.Join(u.Path, filepath.Join(indexKey...)) err = os.MkdirAll(filepath.Dir(path), os.ModePerm) if err != nil { return errors.Wrapf(err, "failed to save index file (%q)", path) } err = ioutil.WriteFile(path, []byte(indexValue.String()), 0644) if err != nil { return errors.Wrapf(err, "failed to save index file (%q)", path) } else { command.V(2).Infof("wrote testimony (%q) index file (%q)", ft.Path, path) } default: return fmt.Errorf("unexpected index URL scheme (%q)", u.Scheme) } return nil } |
< < < < | | < < | < < < < < < < < < < < < < < < < < | | | | | < | < | | | | | | | | | | > > > > > | > > > > > > | < | > > > > > > > | < > | < > | |
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 .. 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 ... 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 ... 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 |
"src.d10.dev/command" "src.d10.dev/hancock/model" ) var ( // Flags that may be used by both publish and testimony operations. storeFlag command.StringSet ) func validateStoreFlag(def string) error { if len(storeFlag) == 0 && def != "" { storeFlag = append(storeFlag, def) } for _, val := range storeFlag { if val == "" { continue } u, err := url.Parse(val) if err != nil { command.Error(errors.Wrapf(err, "bad index URL (%s)", val)) command.Exit() ................................................................................ return nil } func init() { command.RegisterOperation(command.Operation{ Handler: publishMain, Name: "publish", Syntax: "publish [-store=<url>] [in=<directory] [testimony ...]", Description: "Publish signed manifest to an index. Expects signed activity on stdin, as encoded by `hancock-sign`.", }) } func publishMain() error { cfg, err := command.Config() command.Check(err) // defaults cfgTop := cfg.Section("") cfgOp := cfg.Section("publish") defaultStore := cfgOp.Key("store").MustString(cfgTop.Key("store").MustString("https://hancock.beyondcentral.com")) defaultDirectory := cfgOp.Key("testimony").MustString(cfgTop.Key("testimony").String()) // args command.OperationFlagSet.Var(&storeFlag, "store", fmt.Sprintf("URL of testimony store (default %q)", defaultStore)) inFlag := command.OperationFlagSet.String("in", defaultDirectory, "directory containing testimony files") // parse flags err = command.OperationFlagSet.Parse(command.Args()[1:]) if err != nil { return err } // validate flags err = validateStoreFlag(defaultStore) if err != nil { return err } if len(storeFlag) == 0 { return errors.New("Storage URL required. Use flag `-store <URL>` or see configuration file.") } if len(command.OperationFlagSet.Args()) == 0 && *inFlag == "" { return errors.New("Testimony required. Use `-in <directory>` or specify testimony files.") } if *inFlag != "" { inStat, err := os.Stat(*inFlag) ................................................................................ } err = testimony.FileManifest.Check() if err != nil { return err } // validate testimony here? stored := false for _, storeURL := range storeFlag { err = publishTestimonyToStore(storeURL, testimony) if err != nil { command.Error(err) } else { stored = true } } if stored { count++ } return err }) command.V(1).Infof("Published %d items from %q.", count, fileOrDir) if err != nil { command.Error(err) ................................................................................ func publishTestimonyToIPFS(ipfsURL string, ft fileTestimony) error { err := ipfsAdd(ipfsURL, ft.Bytes(), ft.FileName()) return err } // publish testimony either to local filesystem or via POST to online index. func publishTestimonyToStore(storeURL string, ft fileTestimony) error { u, err := url.Parse(storeURL) if err != nil { return err } err = ft.Check() if err != nil { return err } switch u.Scheme { case "http", "https": // post to URL urlPath := storeURL + "/testimony" resp, err := http.Post(urlPath, "application/json", bytes.NewBuffer(ft.Bytes())) if err != nil { return err } defer resp.Body.Close() switch resp.StatusCode { case http.StatusOK, http.StatusCreated, http.StatusAccepted: // success if command.V(2) { command.Infof("posted testimony to %q", urlPath) b, _ := ioutil.ReadAll(resp.Body) command.Info(resp.StatusCode) command.Info(string(b)) } default: command.V(1).Infof("Failed to post testimony: %s", string(ft.Bytes())) b, _ := ioutil.ReadAll(resp.Body) return fmt.Errorf("failed to publish testimony (%q) to index (%s): %s %q", ft.Path, storeURL, resp.Status, string(b)) } case "file": // save to directory key, err := model.IndexKey(ft.Testimony.Content, ft.Authority) if err != nil { return err } // filepath.Join() turns the key values into directories. If we // used flat file names, we might exceed filesystem limits for // number of files. path := filepath.Join(u.Path, filepath.Join(key...)) err = os.MkdirAll(filepath.Dir(path), os.ModePerm) if err != nil { return fmt.Errorf("failed to store testimony file (%q): %w", path, err) } err = ioutil.WriteFile(path, ft.Bytes(), 0644) if err != nil { return fmt.Errorf("failed to store testimony file (%q): %w", path, err) } else { command.V(2).Infof("wrote testimony (%q) to file (%q)", ft.Path, path) } default: return fmt.Errorf("unexpected index URL scheme (%q)", u.Scheme) } return nil } |
Changes to cmd/hancock/testimony.go.
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
...
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
|
// to an index. package main import ( "encoding/json" "fmt" "io" "io/ioutil" "log" "os" "path/filepath" "github.com/pkg/errors" "golang.org/x/crypto/ssh" "src.d10.dev/command" "src.d10.dev/hancock/model" ) func init() { command.RegisterOperation(command.Operation{ Handler: testimonyMain, Name: "testimony", Syntax: "testimony [-out <directory>] [-index=<url>] [-ipfs=<url>]", Description: "Publish signed manifest to an index. Expects signed activity on stdin, as encoded by `hancock-sign`.", }) } // fileTestimony bundles manifest and signature, including (redundant) // encoding of manifest to ensure its CID does not change. type fileTestimony struct { model.Testimony model.FileManifest `json:"manifest"` // calculate these once, use when publishing to multiple indexes and IPFS. encoded []byte filename string } func (this *fileTestimony) FileName() string { if this.filename == "" { pub, err := this.PublicKey() if err != nil { log.Panic(err) } this.filename = fmt.Sprintf("%s-%x.testimony", this.CID, pub.Marshal()) // convention: CID of source, pubkey of sig. } return this.filename } func (this *fileTestimony) Bytes() []byte { if this.encoded == nil { var err error this.encoded, err = model.Encode(this) if err != nil { log.Panic(err) } } return this.encoded } func testimonyMain() error { cfg, err := command.Config() command.Check(err) // defaults cfgTop := cfg.Section("") cfgTestimony := cfg.Section("testimony") defaultDirectory := cfgTestimony.Key("directory").MustString(cfgTop.Key("testimony").String()) defaultIndex := cfgTestimony.Key("index").MustString(cfgTop.Key("index").String()) defaultIPFS := cfgTestimony.Key("ipfs").MustString(cfgTop.Key("ipfs").String()) // define args outFlag := command.OperationFlagSet.String("out", defaultDirectory, "directory where testimony files are written") command.OperationFlagSet.Var(&indexFlag, "index", fmt.Sprintf("URL of testimony index (default %s)", defaultIndex)) ipfsFlag = command.OperationFlagSet.String("ipfs", defaultIPFS, "add testimony via IPFS gateway") noopFlag := command.OperationFlagSet.Bool("n", false, "do not save or publish") // parse args err = command.OperationFlagSet.Parse(command.Args()[1:]) if err != nil { return err } // validate args if *outFlag == "" { return errors.New("Output directory required. Use `-out <directory>`") } outStat, err := os.Stat(*outFlag) if os.IsNotExist(err) { return fmt.Errorf("Output directory does not exist (%q).", *outFlag) } if !outStat.IsDir() { return fmt.Errorf("Not a directory (%q).", *outFlag) } err = validateIndexFlag(defaultIndex) if err != nil { return err } err = validateIPFSFlag() if err != nil { return err } // decode input dec := json.NewDecoder(os.Stdin) // first input, the public key (corresponding to signing private key) var publicEncoded string err = dec.Decode(&publicEncoded) command.Check(err) // ensure key parses public, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicEncoded)) // NOT ssh.ParsePublicKey() command.Check(err) _ = public // may be used in filename // following the public key, we read pairs; each pair a manifest followed by testimony for { var rawMan json.RawMessage err = dec.Decode(&rawMan) ................................................................................ command.Check(man.Check()) var sig ssh.Signature err = dec.Decode(&sig) command.Check(err) testimony := model.Testimony{ Public: publicEncoded, Encoded: rawMan, Signature: sig, } if !*noopFlag { ft := fileTestimony{Testimony: testimony, FileManifest: man} // testimony with decoded manifest if *outFlag != "" { filepath := filepath.Join(*outFlag, ft.FileName()) err = ioutil.WriteFile(filepath, ft.Bytes(), 0644) command.Check(err) command.V(2).Infof("%q wrote %s", man.Path, filepath) } if *ipfsFlag != "" { err = publishTestimonyToIPFS(*ipfsFlag, ft) if err != nil { command.Error(errors.Wrapf(err, "%q IPFS add failed (%s)", man.Path, *ipfsFlag)) } } for _, idxURL := range indexFlag { if idxURL == "" { continue // explicit -index="" on command line } err = publishTestimonyToIndex(idxURL, ft) if err != nil { command.Errorf("%q failed to publish testimony to index (%s): %s", man.Path, idxURL, err) } else { command.V(2).Infof("%q testimony posted to index (%s)", man.Path, idxURL) } } command.V(1).Infof("%q testimony published", man.Path) } } return nil } |
<
<
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
<
>
|
<
<
<
<
>
|
<
<
<
<
<
<
<
<
<
<
<
|
<
>
|
<
<
|
|
|
|
>
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
|
|
|
>
|
>
|
|
|
|
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
...
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
|
// to an index. package main import ( "encoding/json" "fmt" "io" "log" "os" "github.com/pkg/errors" "golang.org/x/crypto/ssh" "src.d10.dev/command" "src.d10.dev/hancock/model" ) func init() { command.RegisterOperation(command.Operation{ Handler: testimonyMain, Name: "testimony", Syntax: "testimony [-store=<url>]", Description: "Publish signed manifest to a hancockd server or local filesystem. Expects signed activity on stdin, as encoded by `hancock-sign`.", }) } // fileTestimony bundles manifest and signature, including (redundant) // encoding of manifest to ensure its CID does not change. type fileTestimony struct { model.Testimony model.FileManifest `json:"manifest"` // calculate these once, use when publishing to multiple stores. encoded []byte filename string } func (this *fileTestimony) FileName() string { if this.filename == "" { pub, err := this.PublicKey() if err != nil { log.Panic(err) } this.filename = fmt.Sprintf("%s-%x.testimony", this.Testimony.Content, pub.Marshal()) // convention: CID of source, pubkey of sig. } // still used? log.Panicf("fileTestimony.FileName() is still used: %q", this.filename) return this.filename } func (this *fileTestimony) Bytes() []byte { if this.encoded == nil { var err error this.encoded, err = model.Encode(this) if err != nil { log.Panic(err) } } return this.encoded } func (this *fileTestimony) Check() error { err := this.Testimony.Check() if err != nil { return err } return this.FileManifest.Check() } func testimonyMain() error { cfg, err := command.Config() command.Check(err) // defaults cfgTop := cfg.Section("") cfgTestimony := cfg.Section("testimony") defaultStore := cfgTestimony.Key("store").MustString(cfgTop.Key("store").String()) // define args command.OperationFlagSet.Var(&storeFlag, "store", fmt.Sprintf("URL where testimony is saved/published (default %q)", defaultStore)) noopFlag := command.OperationFlagSet.Bool("n", false, "do not save/publish") // parse args err = command.OperationFlagSet.Parse(command.Args()[1:]) if err != nil { return err } // validate args err = validateStoreFlag(defaultStore) if err != nil { return err } if len(storeFlag) == 0 { return errors.New("Storage URL required. Use flag `-store <URL>` or see configuration file.") } // decode input dec := json.NewDecoder(os.Stdin) // first input, the public key (corresponding to signing private key) var authorityEncoded model.Authority err = dec.Decode(&authorityEncoded) command.Check(err) // ensure key parses public, _, _, _, err := ssh.ParseAuthorizedKey([]byte(authorityEncoded)) // NOT ssh.ParsePublicKey() command.Check(err) _ = public // may be used in filename // following the public key, we read pairs; each pair a manifest followed by testimony for { var rawMan json.RawMessage err = dec.Decode(&rawMan) ................................................................................ command.Check(man.Check()) var sig ssh.Signature err = dec.Decode(&sig) command.Check(err) testimony := model.Testimony{ Authority: authorityEncoded, Content: man.Content, Encoded: rawMan, Signature: sig, } if !*noopFlag { ft := fileTestimony{Testimony: testimony, FileManifest: man} // testimony with decoded manifest stored := false for _, storeURL := range storeFlag { if storeURL == "" { continue // explicit -index="" on command line } err = publishTestimonyToStore(storeURL, ft) if err != nil { command.Errorf("%q failed to publish testimony to index (%s): %s", man.Path, storeURL, err) } else { stored = true command.V(2).Infof("%q testimony stored (%s)", man.Path, storeURL) } } if stored { command.V(1).Infof("%q testimony published", man.Path) } } } return nil } |