restructuration

This commit is contained in:
rmanach 2025-01-08 10:55:27 +01:00
parent 708db35214
commit 60111911da
10 changed files with 351 additions and 295 deletions

View File

@ -1,3 +1,3 @@
# librapi
A simple server to store, search and download books.
A simple server to store, search and download resources (books, articles, etc...).

View File

@ -1,14 +1,17 @@
package forms
import (
"bytes"
"errors"
"librapi/services"
"mime/multipart"
"io"
"net/http"
"strconv"
"strings"
"time"
"github.com/rs/zerolog/log"
"librapi/services"
)
const MaxFileSize = 200 // in MB
@ -18,12 +21,14 @@ var (
ErrInvalidPassword = errors.New("password must not be empty")
ErrInvalidCredentials = errors.New("bad credentials")
ErrInvalidName = errors.New("book name must not be empty")
ErrInvalidEditor = errors.New("book editor must not be empty")
ErrInvalidName = errors.New("resource name must not be empty")
ErrInvalidEditor = errors.New("resource editor must not be empty")
ErrInvalidYear = errors.New("invalid year, unable to parse")
ErrInvalidYearRange = errors.New("invalid year, can't be greater than today")
ErrInvalidAuthors = errors.New("must at least contains one author")
ErrFileMaxSizeReached = errors.New("max file size reached, must be <= 200MB")
ErrFileOpen = errors.New("unable to open file from form")
ErrFileUnreadable = errors.New("unable to read the file")
)
type StrList = []string
@ -39,16 +44,17 @@ type FormField[T FormFieldType] struct {
}
type UploadFile struct {
File multipart.File
Header *multipart.FileHeader
filename string
content []byte
size int64
}
func (uf *UploadFile) GetFilename() string {
return uf.Header.Filename
return uf.filename
}
func (uf *UploadFile) CheckSize() error {
if uf.Header.Size > (MaxFileSize << 20) {
if uf.size > (MaxFileSize << 20) {
return ErrFileMaxSizeReached
}
return nil
@ -94,22 +100,37 @@ func UploadFormFromRequest(r *http.Request) UploadForm {
log.Err(errParse).Msg("unable to parse date")
uf.Year.Err = ErrInvalidYear.Error()
}
if year > time.Now().Year() {
log.Error().Msg("bad date range")
uf.Year.Err = ErrInvalidYearRange.Error()
}
uf.Year.Value = year
if kw := r.FormValue(uf.Keywords.Name); kw != "" {
uf.Keywords.Value = strings.Split(kw, ",")
}
uf.sanitize()
file, fileh, err := r.FormFile(uf.File.Name)
if err != nil {
log.Err(err).Msg("unable to get file from form")
uf.File.Err = ErrFileOpen.Error()
return uf
}
defer file.Close()
content, err := io.ReadAll(file)
if err != nil {
log.Err(err).Msg("unable to get read file from form")
uf.File.Err = ErrFileOpen.Error()
return uf
}
uf.File.Value = UploadFile{
File: file,
Header: fileh,
filename: fileh.Filename,
content: content,
size: fileh.Size,
}
if err := uf.File.Value.CheckSize(); err != nil {
@ -159,13 +180,14 @@ func (uf *UploadForm) IsSuccess() bool {
return uf.Method == http.MethodPost && uf.Error == "" && !uf.HasErrors()
}
func (uf *UploadForm) IntoMetadata() *services.BookMetadata {
bm := &services.BookMetadata{
func (uf *UploadForm) IntoResource() *services.Resource {
bm := &services.Resource{
Name: uf.Name.Value,
Editor: uf.Editor.Value,
Authors: uf.Authors.Value,
Year: uint16(uf.Year.Value),
Keywords: nil,
Content: bytes.NewBuffer(uf.File.Value.content),
}
if desc := uf.Description.Value; desc != "" {
@ -179,6 +201,30 @@ func (uf *UploadForm) IntoMetadata() *services.BookMetadata {
return bm
}
func (uf *UploadForm) sanitize() {
uf.Name.Value = strings.TrimSpace(uf.Name.Value)
uf.Editor.Value = strings.TrimSpace(uf.Editor.Value)
uf.Description.Value = strings.TrimSpace(uf.Description.Value)
authors := []string{}
for _, a := range uf.Authors.Value {
if a == "" {
continue
}
authors = append(authors, strings.TrimSpace(a))
}
uf.Authors.Value = authors
keywords := []string{}
for _, k := range uf.Keywords.Value {
if k == "" {
continue
}
keywords = append(keywords, strings.TrimSpace(k))
}
uf.Keywords.Value = keywords
}
type LoginForm struct {
Username FormField[string]
Password FormField[string]
@ -228,12 +274,12 @@ type SearchForm struct {
Search FormField[string]
Error error
Method string
Results []services.BookMetadata
Results []services.Resource
}
func SearchFormFromRequest(r *http.Request) SearchForm {
sf := NewSearchForm()
sf.Search.Value = r.FormValue(sf.Search.Name)
sf.Search.Value = strings.TrimSpace(r.FormValue(sf.Search.Name))
return sf
}

View File

@ -12,18 +12,18 @@ import (
const URL = "/download"
func Handler(bs services.IStore) func(http.ResponseWriter, *http.Request) {
func Handler(s services.IStore) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
getDownload(w, r, bs)
getDownload(w, r, s)
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
}
}
func getDownload(w http.ResponseWriter, r *http.Request, bs services.IStore) {
func getDownload(w http.ResponseWriter, r *http.Request, s services.IStore) {
queryParams := r.URL.Query()
downloadFiles, ok := queryParams["file"]
if !ok {
@ -39,7 +39,7 @@ func getDownload(w http.ResponseWriter, r *http.Request, bs services.IStore) {
}
filename := downloadFiles[0]
filePath := filepath.Join(bs.GetStoreDir(), filename)
filePath := filepath.Join(s.GetDir(), filename)
if _, err := os.Stat(filePath); err != nil {
if os.IsNotExist(err) {

View File

@ -57,7 +57,11 @@ func postUploadFile(w http.ResponseWriter, r *http.Request, a services.IAuthenti
filename := uf.File.Value.GetFilename()
log.Info().Str("filename", filename).Msg("file is uploading...")
if err := s.Save(uf.IntoMetadata(), uf.File.Value.File); err != nil {
resource := uf.IntoResource()
if err := s.Create(resource); err != nil {
uf.Error = err.Error()
tmpl, err := templates.ExecuteUploadFormTmpl(&uf, w)
if err != nil {
log.Err(err).Msg("unable to generate upload template")

View File

@ -57,7 +57,7 @@ func main() {
defer fnCancel()
auth := services.NewAuthentication(ctx, isSecure)
bs := services.NewBookStore(storeDir())
bs := services.NewStore(storeDir())
srv := server.NewServer(
ctx,

View File

@ -1,269 +0,0 @@
package services
import (
"database/sql"
_ "embed"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
_ "github.com/mattn/go-sqlite3"
"github.com/rs/zerolog/log"
)
var (
ErrFileCreation = errors.New("unexpected error occurred while creating book file")
ErrFileCopy = errors.New("unexpected error occurred while copying book file")
)
type IStore interface {
Save(bm *BookMetadata, content io.ReadCloser) error
Search(value string) ([]BookMetadata, error)
GetStoreDir() string
}
var _ IStore = (*BookStore)(nil)
const (
sqlSearchBooks = `
select
b.*
from
books b
where
lower(b.name) like lower(?)
or lower(b.description) like lower(?)
or lower(b.editor) like lower(?)
or lower(b.year) like lower(?)
union
select
b2.*
from books b2, json_each(b2.authors)
where lower(json_each.value) like lower(?)
union
select
b3.*
from books b3, json_each(b3.keywords)
where lower(json_each.value) like lower(?)
`
sqlInsertBook = "insert into books(name, description, editor, authors, year, keywords, path) values (?,?,?,?,?,?,?)"
sqlCreateBookTable = `
create table if not exists books (
name text primary key,
description text,
editor text not null,
authors jsonb not null,
year int not null,
keywords jsonb,
path text not null
)
`
)
type BookMetadata struct {
Name string
Description *string
Editor string
Authors []string
Year uint16
Keywords []string
path string
}
func (bm *BookMetadata) getAuthors() (string, error) {
authors, err := json.Marshal(bm.Authors)
if err != nil {
log.Err(err).Msg("unable to marshal authors")
return "", err
}
return string(authors), nil
}
func (bm *BookMetadata) getKeywords() (*string, error) {
if bm.Keywords == nil {
return nil, nil
}
keywords, err := json.Marshal(bm.Keywords)
if err != nil {
log.Err(err).Msg("unable to marshal keywords")
return nil, err
}
strKeywords := string(keywords)
return &strKeywords, nil
}
func (bm *BookMetadata) getFormattedName() string {
name := strings.ReplaceAll(strings.ToLower(bm.Name), " ", "-")
editor := strings.ReplaceAll(strings.ToLower(bm.Editor), " ", "-")
author := strings.ReplaceAll(strings.ToLower(bm.Authors[0]), " ", "-")
return fmt.Sprintf("%s_%s_%s_%d.pdf", name, editor, author, bm.Year)
}
func (bm *BookMetadata) intoStmtValues() ([]any, error) {
authors, err := bm.getAuthors()
if err != nil {
return nil, err
}
keywords, err := bm.getKeywords()
if err != nil {
log.Err(err).Msg("unable to marshal authors")
return nil, err
}
return []any{
bm.Name,
bm.Description,
bm.Editor,
authors,
bm.Year,
keywords,
bm.path,
}, nil
}
func (bm *BookMetadata) GetPath() string {
return bm.path
}
type BookStore struct {
processing *sync.WaitGroup
dir string
db *sql.DB
}
func NewBookStore(dir string) *BookStore {
if err := os.MkdirAll(dir, 0755); err != nil { //nolint
log.Fatal().Err(err).Msg("unable to create store dir")
}
bs := &BookStore{
processing: &sync.WaitGroup{},
dir: dir,
}
bs.init(dir)
return bs
}
func (bs *BookStore) init(dir string) {
db, err := sql.Open("sqlite3", filepath.Join(dir, "librapi.db"))
if err != nil {
log.Fatal().Err(err).Msg("unable initialize sqlite3 database")
}
if _, err := db.Exec(sqlCreateBookTable); err != nil {
log.Fatal().Err(err).Msg("unable to create books table")
}
bs.db = db
}
func (bs *BookStore) store(bm *BookMetadata) error {
stmt, err := bs.db.Prepare(sqlInsertBook)
if err != nil {
log.Err(err).Msg("unable to parse insert book sql")
return err
}
defer stmt.Close()
values, err := bm.intoStmtValues()
if err != nil {
return err
}
if _, err := stmt.Exec(values...); err != nil {
log.Err(err).Msg("unable to insert new book")
return err
}
return nil
}
func (bs *BookStore) Done() <-chan struct{} {
chDone := make(chan struct{})
go func() {
bs.processing.Wait()
if err := bs.db.Close(); err != nil {
log.Err(err).Msg("unable to close the database")
}
chDone <- struct{}{}
log.Info().Msg("book store processing done")
}()
return chDone
}
func (bs *BookStore) GetStoreDir() string {
return bs.dir
}
func (bs *BookStore) Save(bm *BookMetadata, content io.ReadCloser) error {
bs.processing.Add(1)
defer bs.processing.Done()
defer content.Close()
bm.path = filepath.Join(bs.dir, bm.getFormattedName())
dst, err := os.Create(bm.path)
if err != nil {
log.Err(err).Msg(ErrFileCreation.Error())
return ErrFileCreation
}
defer dst.Close()
if _, err := io.Copy(dst, content); err != nil {
log.Err(err).Msg(ErrFileCopy.Error())
return ErrFileCopy
}
return bs.store(bm)
}
func (bs *BookStore) Search(value string) ([]BookMetadata, error) {
stmt, err := bs.db.Prepare(sqlSearchBooks)
if err != nil {
log.Err(err).Msg("unable to parse search book sql")
return nil, err
}
defer stmt.Close()
values := []any{}
for i := 0; i <= 5; i++ {
values = append(values, fmt.Sprintf("%%%s%%", value))
}
rows, err := stmt.Query(values...)
if err != nil {
log.Err(err).Msg("unable to query books")
return nil, err
}
bms := []BookMetadata{}
for rows.Next() {
var bm BookMetadata
var authors string
var keyword *string
if err := rows.Scan(&bm.Name, &bm.Description, &bm.Editor, &authors, &bm.Year, &keyword, &bm.path); err != nil {
log.Err(err).Msg("unable to scan row")
return nil, err
}
var authorsSlice []string
if err := json.Unmarshal([]byte(authors), &authorsSlice); err != nil {
log.Err(err).Msg("unable to unmarshal authors into slice")
return nil, err
}
bm.Authors = authorsSlice
bms = append(bms, bm)
}
return bms, nil
}

275
services/store.go Normal file
View File

@ -0,0 +1,275 @@
package services
import (
"bytes"
"database/sql"
_ "embed"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
_ "github.com/mattn/go-sqlite3"
"github.com/rs/zerolog/log"
)
var (
ErrFileCreation = errors.New("unexpected error occurred while creating resource file")
ErrFileCopy = errors.New("unexpected error occurred while copying resource file")
)
type IStore interface {
Create(r *Resource) error
Search(value string) ([]Resource, error)
GetDir() string
}
var _ IStore = (*Store)(nil)
const (
sqlSearchResources = `
select
r.*
from
resources r
where
lower(r.name) like lower(?)
or lower(r.description) like lower(?)
or lower(r.editor) like lower(?)
or lower(r.year) like lower(?)
union
select
r2.*
from resources r2, json_each(r2.authors)
where lower(json_each.value) like lower(?)
union
select
r3.*
from resources r3, json_each(r3.keywords)
where lower(json_each.value) like lower(?)
`
sqlInsertResource = "insert into resources(name, description, editor, authors, year, keywords, path) values (?,?,?,?,?,?,?)"
sqlCreateResourceTable = `
create table if not exists resources (
name text primary key,
description text,
editor text not null,
authors jsonb not null,
year int not null,
keywords jsonb,
path text not null
)
`
)
func sanitizeFilename(name string) string {
return strings.ReplaceAll(
strings.ReplaceAll(
strings.ToLower(name), " ", "-",
),
"'", "",
)
}
type Resource struct {
Name string
Description *string
Editor string
Authors []string
Year uint16
Keywords []string
Content *bytes.Buffer
Path string
}
func (r *Resource) getAuthors() (string, error) {
authors, err := json.Marshal(r.Authors)
if err != nil {
log.Err(err).Msg("unable to marshal authors")
return "", err
}
return string(authors), nil
}
func (r *Resource) getKeywords() (*string, error) {
if r.Keywords == nil {
return nil, nil
}
keywords, err := json.Marshal(r.Keywords)
if err != nil {
log.Err(err).Msg("unable to marshal keywords")
return nil, err
}
strKeywords := string(keywords)
return &strKeywords, nil
}
func (r *Resource) getFormattedName() string {
name := sanitizeFilename(r.Name)
editor := sanitizeFilename(r.Editor)
author := sanitizeFilename((r.Authors[0]))
return fmt.Sprintf("%s_%s_%s_%d.pdf", name, editor, author, r.Year)
}
func (r *Resource) intoStmtValues() ([]any, error) {
authors, err := r.getAuthors()
if err != nil {
return nil, err
}
keywords, err := r.getKeywords()
if err != nil {
log.Err(err).Msg("unable to marshal authors")
return nil, err
}
return []any{
r.Name,
r.Description,
r.Editor,
authors,
r.Year,
keywords,
r.Path,
}, nil
}
type Store struct {
processing *sync.WaitGroup
dir string
db *sql.DB
}
func NewStore(dir string) *Store {
if err := os.MkdirAll(dir, 0755); err != nil { //nolint
log.Fatal().Err(err).Msg("unable to create store dir")
}
s := &Store{
processing: &sync.WaitGroup{},
dir: dir,
}
s.init(dir)
return s
}
func (s *Store) init(dir string) {
db, err := sql.Open("sqlite3", filepath.Join(dir, "librapi.db"))
if err != nil {
log.Fatal().Err(err).Msg("unable initialize sqlite3 database")
}
if _, err := db.Exec(sqlCreateResourceTable); err != nil {
log.Fatal().Err(err).Msg("unable to create resources table")
}
s.db = db
}
func (s *Store) createRessource(r *Resource) error {
stmt, err := s.db.Prepare(sqlInsertResource)
if err != nil {
log.Err(err).Msg("unable to parse insert resource sql")
return err
}
defer stmt.Close()
values, err := r.intoStmtValues()
if err != nil {
return err
}
if _, err := stmt.Exec(values...); err != nil {
log.Err(err).Msg("unable to insert new resource")
return err
}
return nil
}
func (s *Store) Done() <-chan struct{} {
chDone := make(chan struct{})
go func() {
s.processing.Wait()
if err := s.db.Close(); err != nil {
log.Err(err).Msg("unable to close the database")
}
chDone <- struct{}{}
log.Info().Msg("resource store processing done")
}()
return chDone
}
func (s *Store) GetDir() string {
return s.dir
}
func (s *Store) Create(r *Resource) error {
s.processing.Add(1)
defer s.processing.Done()
path := filepath.Join(s.dir, r.getFormattedName())
r.Path = path
dst, err := os.Create(path)
if err != nil {
log.Err(err).Msg(ErrFileCreation.Error())
return ErrFileCreation
}
defer dst.Close()
if _, err := io.Copy(dst, r.Content); err != nil {
log.Err(err).Msg(ErrFileCopy.Error())
return ErrFileCopy
}
return s.createRessource(r)
}
func (s *Store) Search(value string) ([]Resource, error) {
stmt, err := s.db.Prepare(sqlSearchResources)
if err != nil {
log.Err(err).Msg("unable to parse search resource sql")
return nil, err
}
defer stmt.Close()
values := []any{}
for i := 0; i <= 5; i++ {
values = append(values, fmt.Sprintf("%%%s%%", value))
}
rows, err := stmt.Query(values...)
if err != nil {
log.Err(err).Msg("unable to query resources")
return nil, err
}
resources := []Resource{}
for rows.Next() {
var resource Resource
var authors string
var keyword *string
if err := rows.Scan(&resource.Name, &resource.Description, &resource.Editor, &authors, &resource.Year, &keyword, &resource.Path); err != nil {
log.Err(err).Msg("unable to scan row")
return nil, err
}
var authorsSlice []string
if err := json.Unmarshal([]byte(authors), &authorsSlice); err != nil {
log.Err(err).Msg("unable to unmarshal authors into slice")
return nil, err
}
resource.Authors = authorsSlice
resources = append(resources, resource)
}
return resources, nil
}

View File

@ -1,6 +1,6 @@
{{ define "content" }}
<h3>A simple API to store, search and download books, articles, etc...</h3>
<div>No extra JS, CSS or fancy stuff. You click, you search, you found, you're happy.</div>
<div>No extra JS, CSS or fancy stuff. You click, you search, you download, you're happy.</div>
<form action="/home" method="post" enctype="multipart/form-data">
<div class="main-container">
<div class="form-item">
@ -52,7 +52,7 @@
</td>
<td style="text-align: center">
<div class="col-item">
<a target="_blank" href="{{.GetPath | bookUrl}}">Download</a>
<a target="_blank" href="{{.Path | resourceUrl}}">Download</a>
</div>
</td>
</tr>

View File

@ -68,7 +68,7 @@ var funcMap = template.FuncMap{
}
return ""
},
"bookUrl": func(path string) string {
"resourceUrl": func(path string) string {
_, filename := filepath.Split(path)
return fmt.Sprintf("%s?file=%s", download.URL, filename)
},

View File

@ -1,5 +1,5 @@
{{define "content" }}
<h2>Upload a book</h2>
<h2>Upload a resource</h2>
<form action="/upload" method="post" enctype="multipart/form-data">
<div class="main-container">
<div class="form-item">
@ -58,7 +58,7 @@
</div>
<div class="form-item">
<div class="form-container">
<input type="file" name="file" files="{{.File.Value.Header | filename}}" accept=".pdf" />
<input type="file" name="file" files="{{.File.Value.GetFilename}}" accept=".pdf" />
<button id="submit" type="submit">Upload</button>
</div>
</div>