From 5f46407ff672066c434ed8e2986fad83ef964727 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Wed, 19 Jul 2023 01:48:17 -0500 Subject: [PATCH 01/13] Replace gogs/cron with go-co-op/gocron --- go.mod | 3 ++- go.sum | 10 ++++++++-- services/cron/cron.go | 30 ++++++++++++++++++------------ services/cron/tasks.go | 3 ++- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 9ba54ed185487..ef4645ea5ef23 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/go-ap/jsonld v0.0.0-20221030091449-f2a191312c73 github.com/go-chi/chi/v5 v5.0.8 github.com/go-chi/cors v1.2.1 + github.com/go-co-op/gocron v1.30.1 github.com/go-enry/go-enry/v2 v2.8.4 github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e github.com/go-git/go-billy/v5 v5.4.1 @@ -52,7 +53,6 @@ require ( github.com/go-webauthn/webauthn v0.8.4 github.com/gobwas/glob v0.2.3 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f - github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/go-github/v53 v53.2.0 @@ -254,6 +254,7 @@ require ( github.com/rhysd/actionlint v1.6.25 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/robfig/cron v1.2.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/rs/xid v1.5.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index 5f2704fddbdeb..c532496ff48ce 100644 --- a/go.sum +++ b/go.sum @@ -372,6 +372,8 @@ github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-co-op/gocron v1.30.1 h1:tjWUvJl5KrcwpkEkSXFSQFr4F9h5SfV/m4+RX0cV2fs= +github.com/go-co-op/gocron v1.30.1/go.mod h1:39f6KNSGVOU1LO/ZOoZfcSxwlsJDQOKSu8erN0SH48Y= github.com/go-enry/go-enry/v2 v2.8.4 h1:QrY3hx/RiqCJJRbdU0MOcjfTM1a586J0WSooqdlJIhs= github.com/go-enry/go-enry/v2 v2.8.4/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= @@ -497,8 +499,6 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= -github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 h1:yXtpJr/LV6PFu4nTLgfjQdcMdzjbqqXMEnHfq0Or6p8= -github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14/go.mod h1:jPoNZLWDAqA5N3G5amEoiNbhVrmM+ZQEcnQvNQ2KaZk= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 h1:UjoPNDAQ5JPCjlxoJd6K8ALZqSDDhk2ymieAZOVaDg0= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85/go.mod h1:fR6z1Ie6rtF7kl/vBYMfgD5/G5B1blui7z426/sj2DU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= @@ -786,6 +786,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -1041,10 +1042,14 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= @@ -1251,6 +1256,7 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= diff --git a/services/cron/cron.go b/services/cron/cron.go index 72deb94cebe14..da17aea2af546 100644 --- a/services/cron/cron.go +++ b/services/cron/cron.go @@ -14,10 +14,10 @@ import ( "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/translation" - "github.com/gogs/cron" + "github.com/go-co-op/gocron" ) -var c = cron.New() +var s = gocron.NewScheduler(time.Local) // Prevent duplicate running tasks. var taskStatusTable = sync.NewStatusTable() @@ -39,11 +39,11 @@ func NewContext(original context.Context) { } } - c.Start() + s.StartAsync() started = true lock.Unlock() graceful.GetManager().RunAtShutdown(context.Background(), func() { - c.Stop() + s.Stop() lock.Lock() started = false lock.Unlock() @@ -77,13 +77,19 @@ type TaskTable []*TaskTableRow // ListTasks returns all running cron tasks. func ListTasks() TaskTable { - entries := c.Entries() - eMap := map[string]*cron.Entry{} - for _, e := range entries { - eMap[e.Description] = e + jobs := s.Jobs() + jobMap := map[string]*gocron.Job{} + for _, job := range jobs { + // the first tag is the task name + jobMap[job.Tags()[0]] = job } + lock.Lock() defer lock.Unlock() + for _, job := range jobs { + job.PreviousRun() + } + tTable := make([]*TaskTableRow, 0, len(tasks)) for _, task := range tasks { spec := "-" @@ -91,10 +97,10 @@ func ListTasks() TaskTable { next time.Time prev time.Time ) - if e, ok := eMap[task.Name]; ok { - spec = e.Spec - next = e.Next - prev = e.Prev + if e, ok := jobMap[task.Name]; ok { + spec = e.Tags()[1] // the second tag is the task spec + next = e.NextRun() + prev = e.PreviousRun() } task.lock.Lock() tTable = append(tTable, &TaskTableRow{ diff --git a/services/cron/tasks.go b/services/cron/tasks.go index 1c5493c8983d7..40354fd292115 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -176,7 +176,8 @@ func RegisterTask(name string, config Config, fun func(context.Context, *user_mo if config.IsEnabled() { // We cannot use the entry return as there is no way to lock it - if _, err = c.AddJob(name, config.GetSchedule(), task); err != nil { + tags := []string{name, config.GetSchedule()} // name and schedule can't be get from job, so we add them as tag + if _, err = s.CronWithSeconds(config.GetSchedule()).Tag(tags...).Do(task.Run); err != nil { log.Error("Unable to register cron task with name: %s Error: %v", name, err) return err } From aa4f3fc037c1cbfa7bbd6fe8e057d977dc6e014b Mon Sep 17 00:00:00 2001 From: harryzcy Date: Wed, 19 Jul 2023 01:49:11 -0500 Subject: [PATCH 02/13] Run `make tidy` --- assets/go-licenses.json | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index bd541e7e904b8..c4b6734e579e2 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -409,6 +409,11 @@ "path": "github.com/go-chi/cors/LICENSE", "licenseText": "Copyright (c) 2014 Olivier Poitrey \u003crs@dailymotion.com\u003e\nCopyright (c) 2016-Present https://github.com/go-chi authors\n\nMIT License\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, + { + "name": "github.com/go-co-op/gocron", + "path": "github.com/go-co-op/gocron/LICENSE", + "licenseText": "MIT License\n\nCopyright (c) 2014, 辣椒面\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, { "name": "github.com/go-enry/go-enry/v2", "path": "github.com/go-enry/go-enry/v2/LICENSE", @@ -484,11 +489,6 @@ "path": "github.com/gogs/chardet/LICENSE", "licenseText": "Copyright (c) 2012 chardet Authors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies\nof the Software, and to permit persons to whom the Software is furnished to do\nso, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\nPartial of the Software is derived from ICU project. See icu-license.html for\nlicense of the derivative portions.\n" }, - { - "name": "github.com/gogs/cron", - "path": "github.com/gogs/cron/LICENSE", - "licenseText": "Copyright (C) 2012 Rob Figueiredo\nAll Rights Reserved.\n\nMIT LICENSE\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" - }, { "name": "github.com/gogs/go-gogs-client", "path": "github.com/gogs/go-gogs-client/LICENSE", @@ -914,6 +914,11 @@ "path": "github.com/robfig/cron/LICENSE", "licenseText": "Copyright (C) 2012 Rob Figueiredo\nAll Rights Reserved.\n\nMIT LICENSE\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, + { + "name": "github.com/robfig/cron/v3", + "path": "github.com/robfig/cron/v3/LICENSE", + "licenseText": "Copyright (C) 2012 Rob Figueiredo\nAll Rights Reserved.\n\nMIT LICENSE\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" + }, { "name": "github.com/rs/xid", "path": "github.com/rs/xid/LICENSE", From bd4123adeabf775518df61bca102e3b7036ae353 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Wed, 19 Jul 2023 17:23:14 -0500 Subject: [PATCH 03/13] Fix --- modules/migration/release.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/migration/release.go b/modules/migration/release.go index f92cf25e7bc5d..ef221b992113b 100644 --- a/modules/migration/release.go +++ b/modules/migration/release.go @@ -37,6 +37,7 @@ type Release struct { Assets []*ReleaseAsset Created time.Time Published time.Time + OriginalID int64 `yaml:"-"` // ID from the upstream syncing source } // GetExternalName ExternalUserMigrated interface From 73d9f4a5124be6e73c1ab107bedd1e13de749387 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Wed, 19 Jul 2023 17:29:30 -0500 Subject: [PATCH 04/13] Revert "Fix" that wrong file is added This reverts commit bd4123adeabf775518df61bca102e3b7036ae353. --- modules/migration/release.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/migration/release.go b/modules/migration/release.go index ef221b992113b..f92cf25e7bc5d 100644 --- a/modules/migration/release.go +++ b/modules/migration/release.go @@ -37,7 +37,6 @@ type Release struct { Assets []*ReleaseAsset Created time.Time Published time.Time - OriginalID int64 `yaml:"-"` // ID from the upstream syncing source } // GetExternalName ExternalUserMigrated interface From 5e1e9258b3789fd62c933b70a62870a0b9196469 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Wed, 19 Jul 2023 17:30:27 -0500 Subject: [PATCH 05/13] Apply correct fix --- services/cron/cron.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/cron/cron.go b/services/cron/cron.go index da17aea2af546..d04b0c3c22dd2 100644 --- a/services/cron/cron.go +++ b/services/cron/cron.go @@ -86,9 +86,6 @@ func ListTasks() TaskTable { lock.Lock() defer lock.Unlock() - for _, job := range jobs { - job.PreviousRun() - } tTable := make([]*TaskTableRow, 0, len(tasks)) for _, task := range tasks { From 7c6208c5d680a0e68201b7c847393edc28223fa0 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Wed, 19 Jul 2023 17:39:37 -0500 Subject: [PATCH 06/13] Auto detect of cron schedule has seconds --- services/cron/tasks.go | 25 ++++++++++++++++++++++++- services/cron/tasks_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 services/cron/tasks_test.go diff --git a/services/cron/tasks.go b/services/cron/tasks.go index 40354fd292115..03135dea3b4fd 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" + "github.com/go-co-op/gocron" ) var ( @@ -177,7 +178,13 @@ func RegisterTask(name string, config Config, fun func(context.Context, *user_mo if config.IsEnabled() { // We cannot use the entry return as there is no way to lock it tags := []string{name, config.GetSchedule()} // name and schedule can't be get from job, so we add them as tag - if _, err = s.CronWithSeconds(config.GetSchedule()).Tag(tags...).Do(task.Run); err != nil { + var scheduler *gocron.Scheduler + if isScheduleWithSeconds(config.GetSchedule()) { + scheduler = s.CronWithSeconds(config.GetSchedule()) + } else { + scheduler = s.Cron(config.GetSchedule()) + } + if _, err = scheduler.Tag(tags...).Do(task.Run); err != nil { log.Error("Unable to register cron task with name: %s Error: %v", name, err) return err } @@ -200,3 +207,19 @@ func RegisterTaskFatal(name string, config Config, fun func(context.Context, *us log.Fatal("Unable to register cron task %s Error: %v", name, err) } } + +func isScheduleWithSeconds(schedule string) bool { + numSpaceBlocks := 0 + previousIsSpace := false + for _, c := range schedule { + if c == ' ' { + if !previousIsSpace { + numSpaceBlocks++ + previousIsSpace = true + } + } else { + previousIsSpace = false + } + } + return numSpaceBlocks == 5 +} diff --git a/services/cron/tasks_test.go b/services/cron/tasks_test.go new file mode 100644 index 0000000000000..c8c1154e54b21 --- /dev/null +++ b/services/cron/tasks_test.go @@ -0,0 +1,29 @@ +package cron + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsScheduleWithSeconds(t *testing.T) { + tests := []struct { + schedule string + hasSecond bool + }{ + {"* * * * * *", true}, + {"* * * * *", false}, + {"5 4 * * *", false}, + {"5 4 * * *", false}, + {"5,8 4 * * *", false}, + {"* * * * * *", true}, + {"5,8 4 * * *", false}, + } + + for i, test := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + assert.Equal(t, test.hasSecond, isScheduleWithSeconds(test.schedule)) + }) + } +} From a132628244fd6aa8f42d827b37fa2838fa20f102 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Wed, 19 Jul 2023 18:06:30 -0500 Subject: [PATCH 07/13] Add some tests --- services/cron/cron.go | 9 ++++++++- services/cron/tasks.go | 27 +++++++++++++++++---------- services/cron/tasks_test.go | 33 +++++++++++++++++++++++++++++++-- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/services/cron/cron.go b/services/cron/cron.go index d04b0c3c22dd2..13052d3204b53 100644 --- a/services/cron/cron.go +++ b/services/cron/cron.go @@ -81,6 +81,10 @@ func ListTasks() TaskTable { jobMap := map[string]*gocron.Job{} for _, job := range jobs { // the first tag is the task name + tags := job.Tags() + if len(tags) == 0 { // should never happen + continue + } jobMap[job.Tags()[0]] = job } @@ -95,7 +99,10 @@ func ListTasks() TaskTable { prev time.Time ) if e, ok := jobMap[task.Name]; ok { - spec = e.Tags()[1] // the second tag is the task spec + tags := e.Tags() + if len(tags) > 1 { + spec = tags[1] // the second tag is the task spec + } next = e.NextRun() prev = e.PreviousRun() } diff --git a/services/cron/tasks.go b/services/cron/tasks.go index 03135dea3b4fd..d85fa6bfe9d42 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -177,15 +177,7 @@ func RegisterTask(name string, config Config, fun func(context.Context, *user_mo if config.IsEnabled() { // We cannot use the entry return as there is no way to lock it - tags := []string{name, config.GetSchedule()} // name and schedule can't be get from job, so we add them as tag - var scheduler *gocron.Scheduler - if isScheduleWithSeconds(config.GetSchedule()) { - scheduler = s.CronWithSeconds(config.GetSchedule()) - } else { - scheduler = s.Cron(config.GetSchedule()) - } - if _, err = scheduler.Tag(tags...).Do(task.Run); err != nil { - log.Error("Unable to register cron task with name: %s Error: %v", name, err) + if err := addTaskToScheduler(task); err != nil { return err } } @@ -208,7 +200,22 @@ func RegisterTaskFatal(name string, config Config, fun func(context.Context, *us } } -func isScheduleWithSeconds(schedule string) bool { +func addTaskToScheduler(task *Task) error { + tags := []string{task.Name, task.config.GetSchedule()} // name and schedule can't be get from job, so we add them as tag + var scheduler *gocron.Scheduler + if scheduleHasSeconds(task.config.GetSchedule()) { + scheduler = s.CronWithSeconds(task.config.GetSchedule()) + } else { + scheduler = s.Cron(task.config.GetSchedule()) + } + if _, err := scheduler.Tag(tags...).Do(task.Run); err != nil { + log.Error("Unable to register cron task with name: %s Error: %v", task.Name, err) + return err + } + return nil +} + +func scheduleHasSeconds(schedule string) bool { numSpaceBlocks := 0 previousIsSpace := false for _, c := range schedule { diff --git a/services/cron/tasks_test.go b/services/cron/tasks_test.go index c8c1154e54b21..0e0ed80ad5ff0 100644 --- a/services/cron/tasks_test.go +++ b/services/cron/tasks_test.go @@ -7,7 +7,36 @@ import ( "github.com/stretchr/testify/assert" ) -func TestIsScheduleWithSeconds(t *testing.T) { +func TestAddTaskToScheduler(t *testing.T) { + assert.Len(t, s.Jobs(), 0) + defer s.Clear() + + // no seconds + err := addTaskToScheduler(&Task{ + Name: "task 1", + config: &BaseConfig{ + Schedule: "5 4 * * *", + }, + }) + assert.NoError(t, err) + assert.Len(t, s.Jobs(), 1) + assert.Equal(t, "task 1", s.Jobs()[0].Tags()[0]) + assert.Equal(t, "5 4 * * *", s.Jobs()[0].Tags()[1]) + + // with seconds + err = addTaskToScheduler(&Task{ + Name: "task 2", + config: &BaseConfig{ + Schedule: "30 5 4 * * *", + }, + }) + assert.NoError(t, err) + assert.Len(t, s.Jobs(), 2) + assert.Equal(t, "task 2", s.Jobs()[1].Tags()[0]) + assert.Equal(t, "30 5 4 * * *", s.Jobs()[1].Tags()[1]) +} + +func TestScheduleHasSeconds(t *testing.T) { tests := []struct { schedule string hasSecond bool @@ -23,7 +52,7 @@ func TestIsScheduleWithSeconds(t *testing.T) { for i, test := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - assert.Equal(t, test.hasSecond, isScheduleWithSeconds(test.schedule)) + assert.Equal(t, test.hasSecond, scheduleHasSeconds(test.schedule)) }) } } From 9c15e776e5b783f0a26f374ad46b12af5a47a43e Mon Sep 17 00:00:00 2001 From: harryzcy Date: Wed, 19 Jul 2023 18:07:39 -0500 Subject: [PATCH 08/13] Add copyright --- services/cron/tasks_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/cron/tasks_test.go b/services/cron/tasks_test.go index 0e0ed80ad5ff0..84c7dddd11d03 100644 --- a/services/cron/tasks_test.go +++ b/services/cron/tasks_test.go @@ -1,3 +1,6 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + package cron import ( From cf5d7488ad75239fb153858c1d54994899ceebbb Mon Sep 17 00:00:00 2001 From: harryzcy Date: Wed, 19 Jul 2023 18:39:34 -0500 Subject: [PATCH 09/13] Run `make fmt` --- services/cron/tasks.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/cron/tasks.go b/services/cron/tasks.go index d85fa6bfe9d42..2d12e89b0401b 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" + "github.com/go-co-op/gocron" ) From 3e376061e96f183b3e17d4096de30e9fb1a1bd58 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 21 Jul 2023 02:25:42 -0500 Subject: [PATCH 10/13] Use `scheduler` variable for readability --- services/cron/cron.go | 8 ++++---- services/cron/tasks.go | 4 ++-- services/cron/tasks_test.go | 16 ++++++++-------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/services/cron/cron.go b/services/cron/cron.go index 13052d3204b53..e3f31d08f0c90 100644 --- a/services/cron/cron.go +++ b/services/cron/cron.go @@ -17,7 +17,7 @@ import ( "github.com/go-co-op/gocron" ) -var s = gocron.NewScheduler(time.Local) +var scheduler = gocron.NewScheduler(time.Local) // Prevent duplicate running tasks. var taskStatusTable = sync.NewStatusTable() @@ -39,11 +39,11 @@ func NewContext(original context.Context) { } } - s.StartAsync() + scheduler.StartAsync() started = true lock.Unlock() graceful.GetManager().RunAtShutdown(context.Background(), func() { - s.Stop() + scheduler.Stop() lock.Lock() started = false lock.Unlock() @@ -77,7 +77,7 @@ type TaskTable []*TaskTableRow // ListTasks returns all running cron tasks. func ListTasks() TaskTable { - jobs := s.Jobs() + jobs := scheduler.Jobs() jobMap := map[string]*gocron.Job{} for _, job := range jobs { // the first tag is the task name diff --git a/services/cron/tasks.go b/services/cron/tasks.go index 2d12e89b0401b..5a96650c18ff5 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -205,9 +205,9 @@ func addTaskToScheduler(task *Task) error { tags := []string{task.Name, task.config.GetSchedule()} // name and schedule can't be get from job, so we add them as tag var scheduler *gocron.Scheduler if scheduleHasSeconds(task.config.GetSchedule()) { - scheduler = s.CronWithSeconds(task.config.GetSchedule()) + scheduler = scheduler.CronWithSeconds(task.config.GetSchedule()) } else { - scheduler = s.Cron(task.config.GetSchedule()) + scheduler = scheduler.Cron(task.config.GetSchedule()) } if _, err := scheduler.Tag(tags...).Do(task.Run); err != nil { log.Error("Unable to register cron task with name: %s Error: %v", task.Name, err) diff --git a/services/cron/tasks_test.go b/services/cron/tasks_test.go index 84c7dddd11d03..69052d739c4ce 100644 --- a/services/cron/tasks_test.go +++ b/services/cron/tasks_test.go @@ -11,8 +11,8 @@ import ( ) func TestAddTaskToScheduler(t *testing.T) { - assert.Len(t, s.Jobs(), 0) - defer s.Clear() + assert.Len(t, scheduler.Jobs(), 0) + defer scheduler.Clear() // no seconds err := addTaskToScheduler(&Task{ @@ -22,9 +22,9 @@ func TestAddTaskToScheduler(t *testing.T) { }, }) assert.NoError(t, err) - assert.Len(t, s.Jobs(), 1) - assert.Equal(t, "task 1", s.Jobs()[0].Tags()[0]) - assert.Equal(t, "5 4 * * *", s.Jobs()[0].Tags()[1]) + assert.Len(t, scheduler.Jobs(), 1) + assert.Equal(t, "task 1", scheduler.Jobs()[0].Tags()[0]) + assert.Equal(t, "5 4 * * *", scheduler.Jobs()[0].Tags()[1]) // with seconds err = addTaskToScheduler(&Task{ @@ -34,9 +34,9 @@ func TestAddTaskToScheduler(t *testing.T) { }, }) assert.NoError(t, err) - assert.Len(t, s.Jobs(), 2) - assert.Equal(t, "task 2", s.Jobs()[1].Tags()[0]) - assert.Equal(t, "30 5 4 * * *", s.Jobs()[1].Tags()[1]) + assert.Len(t, scheduler.Jobs(), 2) + assert.Equal(t, "task 2", scheduler.Jobs()[1].Tags()[0]) + assert.Equal(t, "30 5 4 * * *", scheduler.Jobs()[1].Tags()[1]) } func TestScheduleHasSeconds(t *testing.T) { From ac9049882a1016f2b31ec723665269941144788d Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 22 Jul 2023 15:54:02 -0500 Subject: [PATCH 11/13] Fix variable shadowing issue --- services/cron/tasks.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/cron/tasks.go b/services/cron/tasks.go index 5a96650c18ff5..3b618c6ee0a0e 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -17,8 +17,6 @@ import ( "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" - - "github.com/go-co-op/gocron" ) var ( @@ -203,7 +201,6 @@ func RegisterTaskFatal(name string, config Config, fun func(context.Context, *us func addTaskToScheduler(task *Task) error { tags := []string{task.Name, task.config.GetSchedule()} // name and schedule can't be get from job, so we add them as tag - var scheduler *gocron.Scheduler if scheduleHasSeconds(task.config.GetSchedule()) { scheduler = scheduler.CronWithSeconds(task.config.GetSchedule()) } else { From b485bf8745c218ec7e6af2710b02d8c7b2344c01 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 23 Jul 2023 13:05:12 -0500 Subject: [PATCH 12/13] Update services/cron/tasks.go Co-authored-by: delvh --- services/cron/tasks.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/services/cron/tasks.go b/services/cron/tasks.go index 3b618c6ee0a0e..5295049294cd5 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -214,17 +214,5 @@ func addTaskToScheduler(task *Task) error { } func scheduleHasSeconds(schedule string) bool { - numSpaceBlocks := 0 - previousIsSpace := false - for _, c := range schedule { - if c == ' ' { - if !previousIsSpace { - numSpaceBlocks++ - previousIsSpace = true - } - } else { - previousIsSpace = false - } - } - return numSpaceBlocks == 5 + return len(strings.Fields(schedule)) >= 6 } From 7df958ec07f7e51dd11be1c844b01727aceec201 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sun, 23 Jul 2023 13:16:37 -0500 Subject: [PATCH 13/13] Add "strings" import --- services/cron/tasks.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/cron/tasks.go b/services/cron/tasks.go index 5295049294cd5..ea1925c26c738 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "reflect" + "strings" "sync" "code.gitea.io/gitea/models/db"