fix(typesense): centralize sentinel and improve indexing

docs/null-date-sentinel.md

@@ -0,0 +1,15 @@
+# Null Date Sentinel
+
+Vikunja stores dates in Typesense as Unix timestamps. Because Typesense does not support filtering for missing values, tasks without a due date use a sentinel value of `-1` when indexed. Queries that need to include tasks with no due date can use a boolean OR filter:
+
+```
+filter_by=due_date:=-1 || due_date:>=<epoch>
+```
+
+The sentinel is chosen from the negative range so it will never conflict with real timestamps which are always positive.
+
+After upgrading, rebuild the index with:
+
+```
+vikunja reindex-sentinel
+```

go.mod

@@ -61,6 +61,7 @@ require (
 	github.com/redis/go-redis/v9 v9.11.0
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/samedi/caldav-go v3.0.0+incompatible
+	github.com/schollz/progressbar/v3 v3.13.0
 	github.com/spf13/afero v1.14.0
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.20.1
@@ -127,6 +128,7 @@ require (
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect

go.sum

@@ -270,6 +270,7 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm
 github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6 h1:SwcnSwBR7X/5EHJQlXBockkJVIMRVt5yKaesBPMtyZQ=
 github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6/go.mod h1:WrYiIuiXUMIvTDAQw97C+9l0CnBmCcvosPjN3XDqS/o=
 github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -323,8 +324,10 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
@@ -333,6 +336,8 @@ github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEu
 github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
 github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -393,6 +398,7 @@ github.com/redis/go-redis/v9 v9.11.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
@@ -409,6 +415,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
 github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/schollz/progressbar/v3 v3.13.0 h1:9TeeWRcjW2qd05I8Kf9knPkW4vLM/hYoa6z9ABvxje8=
+github.com/schollz/progressbar/v3 v3.13.0/go.mod h1:ZBYnSuLAX2LU8P8UiKN/KgF2DY58AJC8yfVYLPC8Ly4=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@@ -585,6 +593,7 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -598,6 +607,7 @@ golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXct
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=

pkg/cmd/index.go

@@ -57,7 +57,7 @@ var indexCmd = &cobra.Command{
 			}
 		} else {
 			log.Infof("Indexing all tasks… This may take a while.")
-			err = models.ReindexAllTasks()
+			err = models.ReindexAllTasks(nil)
 			if err != nil {
 				log.Criticalf("Could not reindex all tasks into Typesense: %s", err.Error())
 				return

pkg/cmd/reindex_sentinel.go

@@ -0,0 +1,58 @@
+// Vikunja is a to-do list application to facilitate your life.
+// Copyright 2018-present Vikunja and contributors. All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"code.vikunja.io/api/pkg/config"
+	"code.vikunja.io/api/pkg/initialize"
+	"code.vikunja.io/api/pkg/log"
+	"code.vikunja.io/api/pkg/models"
+	"github.com/schollz/progressbar/v3"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(reindexSentinelCmd)
+}
+
+var reindexSentinelCmd = &cobra.Command{
+	Use:   "reindex-sentinel",
+	Short: "Reindex all tasks with sentinel dates",
+	Long:  "Rebuilds the Typesense index using the -1 sentinel for tasks without a due date.",
+	PreRun: func(_ *cobra.Command, _ []string) {
+		initialize.FullInitWithoutAsync()
+	},
+	Run: func(_ *cobra.Command, _ []string) {
+		if config.TypesenseURL.GetString() == "" {
+			log.Error("Typesense not configured")
+			return
+		}
+
+		if err := models.CreateTypesenseCollections(); err != nil {
+			log.Criticalf("Could not create Typesense collections: %s", err.Error())
+			return
+		}
+
+		bar := progressbar.Default(-1)
+		if err := models.ReindexAllTasks(bar); err != nil {
+			log.Criticalf("Could not reindex all tasks: %s", err.Error())
+			return
+		}
+		log.Infof("Done!")
+	},
+}

pkg/models/listeners.go

@@ -536,7 +536,7 @@ func (l *AddTaskToTypesense) Handle(msg *message.Message) (err error) {
 	task := make(map[int64]*Task, 1)
 	task[event.Task.ID] = event.Task // Will be filled with all data by the Typesense connector
 
-	return reindexTasksInTypesense(s, task)
+	return reindexTasksInTypesense(s, task, nil)
 }
 
 // UpdateTaskInTypesense  represents a listener
@@ -562,7 +562,7 @@ func (l *UpdateTaskInTypesense) Handle(msg *message.Message) (err error) {
 	task := make(map[int64]*Task, 1)
 	task[event.Task.ID] = event.Task // Will be filled with all data by the Typesense connector
 
-	return reindexTasksInTypesense(s, task)
+	return reindexTasksInTypesense(s, task, nil)
 }
 
 // UpdateTaskPositionsInTypesense  represents a listener
@@ -597,7 +597,7 @@ func (l *UpdateTaskPositionsInTypesense) Handle(msg *message.Message) (err error
 		taskMap[task.ID] = task
 	}
 
-	return reindexTasksInTypesense(s, taskMap)
+	return reindexTasksInTypesense(s, taskMap, nil)
 }
 
 // IncreaseAttachmentCounter  represents a listener
@@ -747,7 +747,7 @@ func (l *UpdateTaskInSavedFilterViews) Handle(msg *message.Message) (err error)
 		task := make(map[int64]*Task, 1)
 		task[event.Task.ID] = event.Task // Will be filled with all data by the Typesense connector
 
-		return reindexTasksInTypesense(s, task)
+		return reindexTasksInTypesense(s, task, nil)
 	}
 
 	return nil

pkg/models/saved_filters.go

@@ -574,7 +574,7 @@ func upsertRelatedTaskProperties(s *xorm.Session, logPrefix string, newTaskBucke
 			taskMap[t.ID] = t
 		}
 
-		err = reindexTasksInTypesense(s, taskMap)
+		err = reindexTasksInTypesense(s, taskMap, nil)
 		if err != nil {
 			log.Errorf("%sError reindexing tasks into Typesense: %s", logPrefix, err)
 			return

pkg/models/task_search.go

@@ -494,14 +494,14 @@ func convertFilterValues(value interface{}) string {
 
 // Parsing and rebuilding the filter for Typesense has the advantage that we have more control over
 // what Typesense finally gets to see.
-func convertParsedFilterToTypesense(rawFilters []*taskFilter) (filterBy string, err error) {
+func convertParsedFilterToTypesense(rawFilters []*taskFilter, includeNulls bool) (filterBy string, err error) {
 
 	filters := []string{}
 
 	for _, f := range rawFilters {
 
 		if nested, is := f.value.([]*taskFilter); is {
-			nestedDBFilters, err := convertParsedFilterToTypesense(nested)
+			nestedDBFilters, err := convertParsedFilterToTypesense(nested, includeNulls)
 			if err != nil {
 				return "", err
 			}
@@ -562,6 +562,10 @@ func convertParsedFilterToTypesense(rawFilters []*taskFilter) (filterBy string,
 			filter += "]"
 		}
 
+		if f.field == "due_date" && includeNulls {
+			filter = fmt.Sprintf("(due_date:%d || %s)", dueDateSentinel, filter)
+		}
+
 		filters = append(filters, filter)
 	}
 
@@ -592,7 +596,7 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task,
 		projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10))
 	}
 
-	filter, err := convertParsedFilterToTypesense(opts.parsedFilters)
+	filter, err := convertParsedFilterToTypesense(opts.parsedFilters, opts.filterIncludeNulls)
 	if err != nil {
 		return nil, 0, err
 	}

pkg/models/task_search_bench_test.go

@@ -118,7 +118,7 @@ func BenchmarkTaskSearch(b *testing.B) {
 		if err := CreateTypesenseCollections(); err != nil {
 			b.Skipf("typesense server not available: %v", err)
 		}
-		if err := ReindexAllTasks(); err != nil {
+		if err := ReindexAllTasks(nil); err != nil {
 			b.Skipf("typesense server not available: %v", err)
 		}
 	}

pkg/models/tasks.go

@@ -17,7 +17,6 @@
 package models
 
 import (
-	"errors"
 	"math"
 	"regexp"
 	"sort"
@@ -34,9 +33,7 @@ import (
 
 	"dario.cat/mergo"
 	"github.com/google/uuid"
-	clone "github.com/huandu/go-clone/generic"
 	"github.com/jinzhu/copier"
-	"github.com/typesense/typesense-go/v2/typesense"
 	"xorm.io/builder"
 	"xorm.io/xorm"
 )
@@ -305,18 +302,8 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op
 		hasFavoritesProject: hasFavoritesProject,
 	}
 	if config.TypesenseEnabled.GetBool() {
-		var tsSearcher taskSearcher = &typesenseTaskSearcher{
-			s: s,
-		}
-		origOpts := clone.Clone(opts)
+		var tsSearcher taskSearcher = &typesenseTaskSearcher{s: s}
 		tasks, totalItems, err = tsSearcher.Search(opts)
-		// It is possible that project views are not yet in Typesense's index. This causes the query here to fail.
-		// To avoid crashing everything, we fall back to the db search in that case.
-		var tsErr = &typesense.HTTPError{}
-		if err != nil && errors.As(err, &tsErr) && tsErr.Status == 404 {
-			log.Warningf("Unable to fetch tasks from Typesense, error was '%v'. Falling back to db.", err)
-			tasks, totalItems, err = dbSearcher.Search(origOpts)
-		}
 	} else {
 		tasks, totalItems, err = dbSearcher.Search(opts)
 	}