Skip to content

Clean up V3 APIs and organize the code. #597

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pzhao5 wants to merge 5 commits into from
Closed

Clean up V3 APIs and organize the code. #597

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cfe951b
Clean up V3 APIs and organize the code.
Oct 9, 2018
c803e80
Update KVStore to reduce error rate (#596)
pzhao5 Oct 9, 2018
4e8c4c1
Update (#600)
pzhao5 Oct 10, 2018
26c6345
Clean up V3 APIs and organize the code.
Oct 9, 2018
0dfe975
Merge branch 'pzhao122' of github.com:oysterprotocol/brokernode into …
Oct 10, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 44 additions & 36 deletions actions/app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,46 +70,54 @@ func App() *buffalo.App {

app.GET("/metrics", buffalo.WrapHandler(prometheus.Handler()))

apiV2 := app.Group("/api/v2")
apiV3 := app.Group("/api/v3")

// UploadSessions
uploadSessionResourceV2 := actions_v2.UploadSessionResourceV2{}
apiV2.POST("upload-sessions", uploadSessionResourceV2.Create)
apiV2.PUT("upload-sessions/{id}", uploadSessionResourceV2.Update)
apiV2.POST("upload-sessions/beta", uploadSessionResourceV2.CreateBeta)
apiV2.GET("upload-sessions/{id}", uploadSessionResourceV2.GetPaymentStatus)

uploadSessionResourceV3 := actions_v3.UploadSessionResourceV3{}
apiV3.POST("upload-sessions", uploadSessionResourceV2.Create)
apiV3.PUT("upload-sessions/{id}", uploadSessionResourceV3.Update)
apiV3.POST("upload-sessions/beta", uploadSessionResourceV2.CreateBeta)
apiV3.GET("upload-sessions/{id}", uploadSessionResourceV2.GetPaymentStatus)

// Webnodes
webnodeResource := actions_v2.WebnodeResource{}
apiV2.POST("supply/webnodes", webnodeResource.Create)

// Transactions
transactionBrokernodeResource := actions_v2.TransactionBrokernodeResource{}
apiV2.POST("demand/transactions/brokernodes", transactionBrokernodeResource.Create)
apiV2.PUT("demand/transactions/brokernodes/{id}", transactionBrokernodeResource.Update)

transactionGenesisHashResource := actions_v2.TransactionGenesisHashResource{}
apiV2.POST("demand/transactions/genesis_hashes", transactionGenesisHashResource.Create)
apiV2.PUT("demand/transactions/genesis_hashes/{id}", transactionGenesisHashResource.Update)

// Treasures
treasures := actions_v2.TreasuresResource{}
apiV2.POST("treasures", treasures.VerifyAndClaim)

// Status
statusResource := StatusResource{}
apiV2.GET("status", statusResource.CheckStatus)
registerV2Api(app)
registerV3Api(app)
}

oyster_utils.StartProfile()
defer oyster_utils.StopProfile()

return app
}

func registerV2Api(app *buffalo.App) {
apiV2 := app.Group("/api/v2")

// UploadSessions
uploadSessionResourceV2 := actions_v2.UploadSessionResourceV2{}
apiV2.POST("upload-sessions", uploadSessionResourceV2.Create)
apiV2.PUT("upload-sessions/{id}", uploadSessionResourceV2.Update)
apiV2.POST("upload-sessions/beta", uploadSessionResourceV2.CreateBeta)
apiV2.GET("upload-sessions/{id}", uploadSessionResourceV2.GetPaymentStatus)

// Webnodes
webnodeResource := actions_v2.WebnodeResource{}
apiV2.POST("supply/webnodes", webnodeResource.Create)

// Transactions
transactionBrokernodeResource := actions_v2.TransactionBrokernodeResource{}
apiV2.POST("demand/transactions/brokernodes", transactionBrokernodeResource.Create)
apiV2.PUT("demand/transactions/brokernodes/{id}", transactionBrokernodeResource.Update)

transactionGenesisHashResource := actions_v2.TransactionGenesisHashResource{}
apiV2.POST("demand/transactions/genesis_hashes", transactionGenesisHashResource.Create)
apiV2.PUT("demand/transactions/genesis_hashes/{id}", transactionGenesisHashResource.Update)

// Treasures
treasures := actions_v2.TreasuresResource{}
apiV2.POST("treasures", treasures.VerifyAndClaim)

// Status
statusResource := StatusResource{}
apiV2.GET("status", statusResource.CheckStatus)
}

func registerV3Api(app *buffalo.App) {
apiV3 := app.Group("/api/v3")

uploadSessionResourceV3 := actions_v3.UploadSessionResourceV3{}
apiV3.POST("upload-sessions", uploadSessionResourceV2.Create)
apiV3.PUT("upload-sessions/{id}", uploadSessionResourceV3.Update)
apiV3.POST("upload-sessions/beta", uploadSessionResourceV2.CreateBeta)
apiV3.GET("upload-sessions/{id}", uploadSessionResourceV2.GetPaymentStatus)
}
20 changes: 20 additions & 0 deletions actions/v3/actions_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package actions_v3

import (
"github.com/gobuffalo/suite"
"github.com/oysterprotocol/brokernode/services"
"github.com/oysterprotocol/brokernode/utils"
)

type ActionSuite struct {
*suite.Action
}

func (suite *ActionSuite) SetupTest() {
suite.Action.SetupTest()

suite.Nil(oyster_utils.InitKvStore())

EthWrapper = services.EthWrapper
IotaWrapper = services.IotaWrapper
}
4 changes: 0 additions & 4 deletions actions/v3/upload_sessions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,6 @@ type UploadSessionUpdateReqV3 struct {
Chunks []models.ChunkReq `json:"chunks"`
}

func init() {

}

// Update uploads a chunk associated with an upload session.
func (usr *UploadSessionResourceV3) Update(c buffalo.Context) error {
start := PrometheusWrapper.TimeNow()
Expand Down
40 changes: 17 additions & 23 deletions utils/kv_store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,12 +172,11 @@ func buildBadgerName(names []string, separator string) string {
/*InitUniqueKvStore creates a new K:V store associated with a particular upload*/
func InitUniqueKvStore(dbID []string) error {
dbName := GetBadgerDBName(dbID)
dirPath := GetBadgerDirName(dbID)
val, ok := dbMap.Get(dbName)
if ok == true && val != nil {
if _, ok := dbMap.Get(dbName); ok {
return nil
}

dirPath := GetBadgerDirName(dbID)
opts := badger.DefaultOptions

if os.Getenv("GO_ENV") == "test" {
Expand Down Expand Up @@ -240,7 +239,7 @@ func InitKvStore() (err error) {
/*CloseUniqueKvStore closes the K:V store associated with a particular upload.*/
func CloseUniqueKvStore(dbName string) error {
value, ok := dbMap.Get(dbName)
if value == nil || ok != true {
if !ok {
return nil
}
dbData := value.(DBData)
Expand Down Expand Up @@ -269,7 +268,7 @@ func CloseKvStore() error {
/*RemoveAllUniqueKvStoreData removes all the data associated with a particular K:V store.*/
func RemoveAllUniqueKvStoreData(dbName string) error {
value, ok := dbMap.Get(dbName)
if value == nil || ok != true {
if !ok {
return errors.New("did not find database with name: " +
dbName + " in RemoveAllUniqueKvStoreData")
}
Expand Down Expand Up @@ -299,9 +298,8 @@ func RemoveAllKvStoreDataFromAllKvStores() []error {
allDBs := dbMap.Keys()
for _, dbName := range allDBs {
value, ok := dbMap.Get(dbName)
if value == nil || ok != true {
errArray = append(errArray, errors.New("did not find database with name: "+
dbName+" in RemoveAllKvStoreDataFromAllKvStores"))
if !ok {
continue
}
dbData := value.(DBData)
directoryPath := dbData.DirectoryPath
Expand Down Expand Up @@ -342,7 +340,7 @@ func RemoveAllKvStoreData() error {
/*GetUniqueBadgerDb returns a database associated with an upload. If not initialized this will return nil. */
func GetUniqueBadgerDb(dbName string) *badger.DB {
value, ok := dbMap.Get(dbName)
if value == nil || ok != true {
if !ok {
return nil
}
dbData := value.(DBData)
Expand All @@ -369,20 +367,16 @@ func GetOrInitUniqueBadgerDB(dbID []string) *badger.DB {
if db != nil {
return db
}
err = InitUniqueKvStore(dbID)
LogIfError(err, nil)
if err == nil {
if err := InitUniqueKvStore(dbID); err == nil {
break
}
if timesRetried >= timesToRetry {
LogIfError(errors.New("retried InitUniqueKvStore too many times"), nil)
LogIfError(errors.New("retried InitUniqueKvStore too many times"), map[string]interface{}{"RetryCount": timesToRetry, "BadgerKey": dbID})
break
}
timesRetried++
}
}

LogIfError(err, nil)
return GetUniqueBadgerDb(dbName)
}

Expand Down Expand Up @@ -632,7 +626,7 @@ func BatchSetToUniqueDB(dbID []string, kvs *KVPairs, ttl time.Duration) error {

if e == badger.ErrTxnTooBig {
e = nil
if commitErr := txn.Commit(nil); commitErr != nil {
if commitErr := txn.Commit(); commitErr != nil {
e = commitErr
} else {
txn = db.NewTransaction(true)
Expand All @@ -648,7 +642,7 @@ func BatchSetToUniqueDB(dbID []string, kvs *KVPairs, ttl time.Duration) error {

defer txn.Discard()
if err == nil {
err = txn.Commit(nil)
err = txn.Commit()
}
LogIfError(err, map[string]interface{}{"batchSize": len(*kvs)})
return err
Expand Down Expand Up @@ -676,7 +670,7 @@ func BatchSet(kvs *KVPairs, ttl time.Duration) error {

if e == badger.ErrTxnTooBig {
e = nil
if commitErr := txn.Commit(nil); commitErr != nil {
if commitErr := txn.Commit(); commitErr != nil {
e = commitErr
} else {
txn = badgerDB.NewTransaction(true)
Expand All @@ -692,7 +686,7 @@ func BatchSet(kvs *KVPairs, ttl time.Duration) error {

defer txn.Discard()
if err == nil {
err = txn.Commit(nil)
err = txn.Commit()
}
LogIfError(err, map[string]interface{}{"batchSize": len(*kvs)})
return err
Expand Down Expand Up @@ -721,7 +715,7 @@ func BatchDeleteFromUniqueDB(dbID []string, ks *KVKeys) error {

if e == badger.ErrTxnTooBig {
e = nil
if commitErr := txn.Commit(nil); commitErr != nil {
if commitErr := txn.Commit(); commitErr != nil {
e = commitErr
} else {
txn = db.NewTransaction(true)
Expand All @@ -737,7 +731,7 @@ func BatchDeleteFromUniqueDB(dbID []string, ks *KVKeys) error {

defer txn.Discard()
if err == nil {
err = txn.Commit(nil)
err = txn.Commit()
}

LogIfError(err, map[string]interface{}{"batchSize": len(*ks)})
Expand All @@ -760,7 +754,7 @@ func BatchDelete(ks *KVKeys) error {

if e == badger.ErrTxnTooBig {
e = nil
if commitErr := txn.Commit(nil); commitErr != nil {
if commitErr := txn.Commit(); commitErr != nil {
e = commitErr
} else {
txn = badgerDB.NewTransaction(true)
Expand All @@ -776,7 +770,7 @@ func BatchDelete(ks *KVKeys) error {

defer txn.Discard()
if err == nil {
err = txn.Commit(nil)
err = txn.Commit()
}

LogIfError(err, map[string]interface{}{"batchSize": len(*ks)})
Expand Down