From 60111911da31142f899cf11c8c76181ec16c1874 Mon Sep 17 00:00:00 2001 From: rmanach Date: Wed, 8 Jan 2025 10:55:27 +0100 Subject: [PATCH] restructuration --- README.md | 2 +- forms/forms.go | 74 +++++++-- handlers/download/handler.go | 8 +- handlers/upload/handler.go | 6 +- main.go | 2 +- services/book_store.go | 269 -------------------------------- services/store.go | 275 +++++++++++++++++++++++++++++++++ templates/home.html.tpl | 4 +- templates/templates.go | 2 +- templates/upload_form.html.tpl | 4 +- 10 files changed, 351 insertions(+), 295 deletions(-) delete mode 100644 services/book_store.go create mode 100644 services/store.go diff --git a/README.md b/README.md index 4f27c10..dc1d3ea 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # librapi -A simple server to store, search and download books. \ No newline at end of file +A simple server to store, search and download resources (books, articles, etc...). \ No newline at end of file diff --git a/forms/forms.go b/forms/forms.go index b9e73fe..9a27f32 100644 --- a/forms/forms.go +++ b/forms/forms.go @@ -1,14 +1,17 @@ package forms import ( + "bytes" "errors" - "librapi/services" - "mime/multipart" + "io" "net/http" "strconv" "strings" + "time" "github.com/rs/zerolog/log" + + "librapi/services" ) const MaxFileSize = 200 // in MB @@ -18,12 +21,14 @@ var ( ErrInvalidPassword = errors.New("password must not be empty") ErrInvalidCredentials = errors.New("bad credentials") - ErrInvalidName = errors.New("book name must not be empty") - ErrInvalidEditor = errors.New("book editor must not be empty") + ErrInvalidName = errors.New("resource name must not be empty") + ErrInvalidEditor = errors.New("resource editor must not be empty") ErrInvalidYear = errors.New("invalid year, unable to parse") + ErrInvalidYearRange = errors.New("invalid year, can't be greater than today") ErrInvalidAuthors = errors.New("must at least contains one author") ErrFileMaxSizeReached = errors.New("max file size reached, must be <= 200MB") ErrFileOpen = errors.New("unable to open file from form") + ErrFileUnreadable = errors.New("unable to read the file") ) type StrList = []string @@ -39,16 +44,17 @@ type FormField[T FormFieldType] struct { } type UploadFile struct { - File multipart.File - Header *multipart.FileHeader + filename string + content []byte + size int64 } func (uf *UploadFile) GetFilename() string { - return uf.Header.Filename + return uf.filename } func (uf *UploadFile) CheckSize() error { - if uf.Header.Size > (MaxFileSize << 20) { + if uf.size > (MaxFileSize << 20) { return ErrFileMaxSizeReached } return nil @@ -94,22 +100,37 @@ func UploadFormFromRequest(r *http.Request) UploadForm { log.Err(errParse).Msg("unable to parse date") uf.Year.Err = ErrInvalidYear.Error() } + if year > time.Now().Year() { + log.Error().Msg("bad date range") + uf.Year.Err = ErrInvalidYearRange.Error() + } uf.Year.Value = year if kw := r.FormValue(uf.Keywords.Name); kw != "" { uf.Keywords.Value = strings.Split(kw, ",") } + uf.sanitize() + file, fileh, err := r.FormFile(uf.File.Name) if err != nil { log.Err(err).Msg("unable to get file from form") uf.File.Err = ErrFileOpen.Error() return uf } + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + log.Err(err).Msg("unable to get read file from form") + uf.File.Err = ErrFileOpen.Error() + return uf + } uf.File.Value = UploadFile{ - File: file, - Header: fileh, + filename: fileh.Filename, + content: content, + size: fileh.Size, } if err := uf.File.Value.CheckSize(); err != nil { @@ -159,13 +180,14 @@ func (uf *UploadForm) IsSuccess() bool { return uf.Method == http.MethodPost && uf.Error == "" && !uf.HasErrors() } -func (uf *UploadForm) IntoMetadata() *services.BookMetadata { - bm := &services.BookMetadata{ +func (uf *UploadForm) IntoResource() *services.Resource { + bm := &services.Resource{ Name: uf.Name.Value, Editor: uf.Editor.Value, Authors: uf.Authors.Value, Year: uint16(uf.Year.Value), Keywords: nil, + Content: bytes.NewBuffer(uf.File.Value.content), } if desc := uf.Description.Value; desc != "" { @@ -179,6 +201,30 @@ func (uf *UploadForm) IntoMetadata() *services.BookMetadata { return bm } +func (uf *UploadForm) sanitize() { + uf.Name.Value = strings.TrimSpace(uf.Name.Value) + uf.Editor.Value = strings.TrimSpace(uf.Editor.Value) + uf.Description.Value = strings.TrimSpace(uf.Description.Value) + + authors := []string{} + for _, a := range uf.Authors.Value { + if a == "" { + continue + } + authors = append(authors, strings.TrimSpace(a)) + } + uf.Authors.Value = authors + + keywords := []string{} + for _, k := range uf.Keywords.Value { + if k == "" { + continue + } + keywords = append(keywords, strings.TrimSpace(k)) + } + uf.Keywords.Value = keywords +} + type LoginForm struct { Username FormField[string] Password FormField[string] @@ -228,12 +274,12 @@ type SearchForm struct { Search FormField[string] Error error Method string - Results []services.BookMetadata + Results []services.Resource } func SearchFormFromRequest(r *http.Request) SearchForm { sf := NewSearchForm() - sf.Search.Value = r.FormValue(sf.Search.Name) + sf.Search.Value = strings.TrimSpace(r.FormValue(sf.Search.Name)) return sf } diff --git a/handlers/download/handler.go b/handlers/download/handler.go index ddeb69b..29c6973 100644 --- a/handlers/download/handler.go +++ b/handlers/download/handler.go @@ -12,18 +12,18 @@ import ( const URL = "/download" -func Handler(bs services.IStore) func(http.ResponseWriter, *http.Request) { +func Handler(s services.IStore) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: - getDownload(w, r, bs) + getDownload(w, r, s) default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } } } -func getDownload(w http.ResponseWriter, r *http.Request, bs services.IStore) { +func getDownload(w http.ResponseWriter, r *http.Request, s services.IStore) { queryParams := r.URL.Query() downloadFiles, ok := queryParams["file"] if !ok { @@ -39,7 +39,7 @@ func getDownload(w http.ResponseWriter, r *http.Request, bs services.IStore) { } filename := downloadFiles[0] - filePath := filepath.Join(bs.GetStoreDir(), filename) + filePath := filepath.Join(s.GetDir(), filename) if _, err := os.Stat(filePath); err != nil { if os.IsNotExist(err) { diff --git a/handlers/upload/handler.go b/handlers/upload/handler.go index 4b7d77f..1291b8e 100644 --- a/handlers/upload/handler.go +++ b/handlers/upload/handler.go @@ -57,7 +57,11 @@ func postUploadFile(w http.ResponseWriter, r *http.Request, a services.IAuthenti filename := uf.File.Value.GetFilename() log.Info().Str("filename", filename).Msg("file is uploading...") - if err := s.Save(uf.IntoMetadata(), uf.File.Value.File); err != nil { + resource := uf.IntoResource() + + if err := s.Create(resource); err != nil { + uf.Error = err.Error() + tmpl, err := templates.ExecuteUploadFormTmpl(&uf, w) if err != nil { log.Err(err).Msg("unable to generate upload template") diff --git a/main.go b/main.go index a871d14..04daa9f 100644 --- a/main.go +++ b/main.go @@ -57,7 +57,7 @@ func main() { defer fnCancel() auth := services.NewAuthentication(ctx, isSecure) - bs := services.NewBookStore(storeDir()) + bs := services.NewStore(storeDir()) srv := server.NewServer( ctx, diff --git a/services/book_store.go b/services/book_store.go deleted file mode 100644 index 690707f..0000000 --- a/services/book_store.go +++ /dev/null @@ -1,269 +0,0 @@ -package services - -import ( - "database/sql" - _ "embed" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "sync" - - _ "github.com/mattn/go-sqlite3" - "github.com/rs/zerolog/log" -) - -var ( - ErrFileCreation = errors.New("unexpected error occurred while creating book file") - ErrFileCopy = errors.New("unexpected error occurred while copying book file") -) - -type IStore interface { - Save(bm *BookMetadata, content io.ReadCloser) error - Search(value string) ([]BookMetadata, error) - GetStoreDir() string -} - -var _ IStore = (*BookStore)(nil) - -const ( - sqlSearchBooks = ` - select - b.* - from - books b - where - lower(b.name) like lower(?) - or lower(b.description) like lower(?) - or lower(b.editor) like lower(?) - or lower(b.year) like lower(?) - union - select - b2.* - from books b2, json_each(b2.authors) - where lower(json_each.value) like lower(?) - union - select - b3.* - from books b3, json_each(b3.keywords) - where lower(json_each.value) like lower(?) -` - sqlInsertBook = "insert into books(name, description, editor, authors, year, keywords, path) values (?,?,?,?,?,?,?)" - sqlCreateBookTable = ` - create table if not exists books ( - name text primary key, - description text, - editor text not null, - authors jsonb not null, - year int not null, - keywords jsonb, - path text not null - ) - ` -) - -type BookMetadata struct { - Name string - Description *string - Editor string - Authors []string - Year uint16 - Keywords []string - path string -} - -func (bm *BookMetadata) getAuthors() (string, error) { - authors, err := json.Marshal(bm.Authors) - if err != nil { - log.Err(err).Msg("unable to marshal authors") - return "", err - } - return string(authors), nil -} - -func (bm *BookMetadata) getKeywords() (*string, error) { - if bm.Keywords == nil { - return nil, nil - } - - keywords, err := json.Marshal(bm.Keywords) - if err != nil { - log.Err(err).Msg("unable to marshal keywords") - return nil, err - } - strKeywords := string(keywords) - return &strKeywords, nil -} - -func (bm *BookMetadata) getFormattedName() string { - name := strings.ReplaceAll(strings.ToLower(bm.Name), " ", "-") - editor := strings.ReplaceAll(strings.ToLower(bm.Editor), " ", "-") - author := strings.ReplaceAll(strings.ToLower(bm.Authors[0]), " ", "-") - - return fmt.Sprintf("%s_%s_%s_%d.pdf", name, editor, author, bm.Year) -} - -func (bm *BookMetadata) intoStmtValues() ([]any, error) { - authors, err := bm.getAuthors() - if err != nil { - return nil, err - } - - keywords, err := bm.getKeywords() - if err != nil { - log.Err(err).Msg("unable to marshal authors") - return nil, err - } - - return []any{ - bm.Name, - bm.Description, - bm.Editor, - authors, - bm.Year, - keywords, - bm.path, - }, nil -} - -func (bm *BookMetadata) GetPath() string { - return bm.path -} - -type BookStore struct { - processing *sync.WaitGroup - dir string - db *sql.DB -} - -func NewBookStore(dir string) *BookStore { - if err := os.MkdirAll(dir, 0755); err != nil { //nolint - log.Fatal().Err(err).Msg("unable to create store dir") - } - - bs := &BookStore{ - processing: &sync.WaitGroup{}, - dir: dir, - } - - bs.init(dir) - return bs -} - -func (bs *BookStore) init(dir string) { - db, err := sql.Open("sqlite3", filepath.Join(dir, "librapi.db")) - if err != nil { - log.Fatal().Err(err).Msg("unable initialize sqlite3 database") - } - - if _, err := db.Exec(sqlCreateBookTable); err != nil { - log.Fatal().Err(err).Msg("unable to create books table") - } - - bs.db = db -} - -func (bs *BookStore) store(bm *BookMetadata) error { - stmt, err := bs.db.Prepare(sqlInsertBook) - if err != nil { - log.Err(err).Msg("unable to parse insert book sql") - return err - } - defer stmt.Close() - - values, err := bm.intoStmtValues() - if err != nil { - return err - } - - if _, err := stmt.Exec(values...); err != nil { - log.Err(err).Msg("unable to insert new book") - return err - } - - return nil -} - -func (bs *BookStore) Done() <-chan struct{} { - chDone := make(chan struct{}) - go func() { - bs.processing.Wait() - if err := bs.db.Close(); err != nil { - log.Err(err).Msg("unable to close the database") - } - chDone <- struct{}{} - log.Info().Msg("book store processing done") - }() - return chDone -} - -func (bs *BookStore) GetStoreDir() string { - return bs.dir -} - -func (bs *BookStore) Save(bm *BookMetadata, content io.ReadCloser) error { - bs.processing.Add(1) - defer bs.processing.Done() - - defer content.Close() - - bm.path = filepath.Join(bs.dir, bm.getFormattedName()) - - dst, err := os.Create(bm.path) - if err != nil { - log.Err(err).Msg(ErrFileCreation.Error()) - return ErrFileCreation - } - defer dst.Close() - - if _, err := io.Copy(dst, content); err != nil { - log.Err(err).Msg(ErrFileCopy.Error()) - return ErrFileCopy - } - - return bs.store(bm) -} - -func (bs *BookStore) Search(value string) ([]BookMetadata, error) { - stmt, err := bs.db.Prepare(sqlSearchBooks) - if err != nil { - log.Err(err).Msg("unable to parse search book sql") - return nil, err - } - defer stmt.Close() - - values := []any{} - for i := 0; i <= 5; i++ { - values = append(values, fmt.Sprintf("%%%s%%", value)) - } - - rows, err := stmt.Query(values...) - if err != nil { - log.Err(err).Msg("unable to query books") - return nil, err - } - - bms := []BookMetadata{} - for rows.Next() { - var bm BookMetadata - var authors string - var keyword *string - if err := rows.Scan(&bm.Name, &bm.Description, &bm.Editor, &authors, &bm.Year, &keyword, &bm.path); err != nil { - log.Err(err).Msg("unable to scan row") - return nil, err - } - - var authorsSlice []string - if err := json.Unmarshal([]byte(authors), &authorsSlice); err != nil { - log.Err(err).Msg("unable to unmarshal authors into slice") - return nil, err - } - bm.Authors = authorsSlice - bms = append(bms, bm) - } - - return bms, nil -} diff --git a/services/store.go b/services/store.go new file mode 100644 index 0000000..6ad1f17 --- /dev/null +++ b/services/store.go @@ -0,0 +1,275 @@ +package services + +import ( + "bytes" + "database/sql" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + + _ "github.com/mattn/go-sqlite3" + "github.com/rs/zerolog/log" +) + +var ( + ErrFileCreation = errors.New("unexpected error occurred while creating resource file") + ErrFileCopy = errors.New("unexpected error occurred while copying resource file") +) + +type IStore interface { + Create(r *Resource) error + Search(value string) ([]Resource, error) + GetDir() string +} + +var _ IStore = (*Store)(nil) + +const ( + sqlSearchResources = ` + select + r.* + from + resources r + where + lower(r.name) like lower(?) + or lower(r.description) like lower(?) + or lower(r.editor) like lower(?) + or lower(r.year) like lower(?) + union + select + r2.* + from resources r2, json_each(r2.authors) + where lower(json_each.value) like lower(?) + union + select + r3.* + from resources r3, json_each(r3.keywords) + where lower(json_each.value) like lower(?) +` + sqlInsertResource = "insert into resources(name, description, editor, authors, year, keywords, path) values (?,?,?,?,?,?,?)" + sqlCreateResourceTable = ` + create table if not exists resources ( + name text primary key, + description text, + editor text not null, + authors jsonb not null, + year int not null, + keywords jsonb, + path text not null + ) + ` +) + +func sanitizeFilename(name string) string { + return strings.ReplaceAll( + strings.ReplaceAll( + strings.ToLower(name), " ", "-", + ), + "'", "", + ) +} + +type Resource struct { + Name string + Description *string + Editor string + Authors []string + Year uint16 + Keywords []string + Content *bytes.Buffer + Path string +} + +func (r *Resource) getAuthors() (string, error) { + authors, err := json.Marshal(r.Authors) + if err != nil { + log.Err(err).Msg("unable to marshal authors") + return "", err + } + return string(authors), nil +} + +func (r *Resource) getKeywords() (*string, error) { + if r.Keywords == nil { + return nil, nil + } + + keywords, err := json.Marshal(r.Keywords) + if err != nil { + log.Err(err).Msg("unable to marshal keywords") + return nil, err + } + strKeywords := string(keywords) + return &strKeywords, nil +} + +func (r *Resource) getFormattedName() string { + name := sanitizeFilename(r.Name) + editor := sanitizeFilename(r.Editor) + author := sanitizeFilename((r.Authors[0])) + + return fmt.Sprintf("%s_%s_%s_%d.pdf", name, editor, author, r.Year) +} + +func (r *Resource) intoStmtValues() ([]any, error) { + authors, err := r.getAuthors() + if err != nil { + return nil, err + } + + keywords, err := r.getKeywords() + if err != nil { + log.Err(err).Msg("unable to marshal authors") + return nil, err + } + + return []any{ + r.Name, + r.Description, + r.Editor, + authors, + r.Year, + keywords, + r.Path, + }, nil +} + +type Store struct { + processing *sync.WaitGroup + dir string + db *sql.DB +} + +func NewStore(dir string) *Store { + if err := os.MkdirAll(dir, 0755); err != nil { //nolint + log.Fatal().Err(err).Msg("unable to create store dir") + } + + s := &Store{ + processing: &sync.WaitGroup{}, + dir: dir, + } + + s.init(dir) + return s +} + +func (s *Store) init(dir string) { + db, err := sql.Open("sqlite3", filepath.Join(dir, "librapi.db")) + if err != nil { + log.Fatal().Err(err).Msg("unable initialize sqlite3 database") + } + + if _, err := db.Exec(sqlCreateResourceTable); err != nil { + log.Fatal().Err(err).Msg("unable to create resources table") + } + + s.db = db +} + +func (s *Store) createRessource(r *Resource) error { + stmt, err := s.db.Prepare(sqlInsertResource) + if err != nil { + log.Err(err).Msg("unable to parse insert resource sql") + return err + } + defer stmt.Close() + + values, err := r.intoStmtValues() + if err != nil { + return err + } + + if _, err := stmt.Exec(values...); err != nil { + log.Err(err).Msg("unable to insert new resource") + return err + } + + return nil +} + +func (s *Store) Done() <-chan struct{} { + chDone := make(chan struct{}) + go func() { + s.processing.Wait() + if err := s.db.Close(); err != nil { + log.Err(err).Msg("unable to close the database") + } + chDone <- struct{}{} + log.Info().Msg("resource store processing done") + }() + return chDone +} + +func (s *Store) GetDir() string { + return s.dir +} + +func (s *Store) Create(r *Resource) error { + s.processing.Add(1) + defer s.processing.Done() + + path := filepath.Join(s.dir, r.getFormattedName()) + r.Path = path + + dst, err := os.Create(path) + if err != nil { + log.Err(err).Msg(ErrFileCreation.Error()) + return ErrFileCreation + } + defer dst.Close() + + if _, err := io.Copy(dst, r.Content); err != nil { + log.Err(err).Msg(ErrFileCopy.Error()) + return ErrFileCopy + } + + return s.createRessource(r) +} + +func (s *Store) Search(value string) ([]Resource, error) { + stmt, err := s.db.Prepare(sqlSearchResources) + if err != nil { + log.Err(err).Msg("unable to parse search resource sql") + return nil, err + } + defer stmt.Close() + + values := []any{} + for i := 0; i <= 5; i++ { + values = append(values, fmt.Sprintf("%%%s%%", value)) + } + + rows, err := stmt.Query(values...) + if err != nil { + log.Err(err).Msg("unable to query resources") + return nil, err + } + + resources := []Resource{} + for rows.Next() { + var resource Resource + var authors string + var keyword *string + if err := rows.Scan(&resource.Name, &resource.Description, &resource.Editor, &authors, &resource.Year, &keyword, &resource.Path); err != nil { + log.Err(err).Msg("unable to scan row") + return nil, err + } + + var authorsSlice []string + if err := json.Unmarshal([]byte(authors), &authorsSlice); err != nil { + log.Err(err).Msg("unable to unmarshal authors into slice") + return nil, err + } + resource.Authors = authorsSlice + resources = append(resources, resource) + } + + return resources, nil +} diff --git a/templates/home.html.tpl b/templates/home.html.tpl index 69fcded..5b1ccc7 100644 --- a/templates/home.html.tpl +++ b/templates/home.html.tpl @@ -1,6 +1,6 @@ {{ define "content" }}

A simple API to store, search and download books, articles, etc...

-
No extra JS, CSS or fancy stuff. You click, you search, you found, you're happy.
+
No extra JS, CSS or fancy stuff. You click, you search, you download, you're happy.
@@ -52,7 +52,7 @@
- Download + Download
diff --git a/templates/templates.go b/templates/templates.go index 3a380cf..630a8d5 100644 --- a/templates/templates.go +++ b/templates/templates.go @@ -68,7 +68,7 @@ var funcMap = template.FuncMap{ } return "" }, - "bookUrl": func(path string) string { + "resourceUrl": func(path string) string { _, filename := filepath.Split(path) return fmt.Sprintf("%s?file=%s", download.URL, filename) }, diff --git a/templates/upload_form.html.tpl b/templates/upload_form.html.tpl index e9a2c8e..3584fb8 100644 --- a/templates/upload_form.html.tpl +++ b/templates/upload_form.html.tpl @@ -1,5 +1,5 @@ {{define "content" }} -

Upload a book

+

Upload a resource

@@ -58,7 +58,7 @@
- +