|
| 1 | +// Copyright 2023 The Gitea Authors. All rights reserved. |
| 2 | +// SPDX-License-Identifier: MIT |
| 3 | + |
| 4 | +package db |
| 5 | + |
| 6 | +import ( |
| 7 | + "errors" |
| 8 | + "fmt" |
| 9 | + "strings" |
| 10 | + |
| 11 | + "code.gitea.io/gitea/modules/container" |
| 12 | + "code.gitea.io/gitea/modules/log" |
| 13 | + "code.gitea.io/gitea/modules/setting" |
| 14 | + |
| 15 | + "xorm.io/xorm" |
| 16 | + "xorm.io/xorm/schemas" |
| 17 | +) |
| 18 | + |
| 19 | +type CheckCollationsResult struct { |
| 20 | + ExpectedCollation string |
| 21 | + AvailableCollation container.Set[string] |
| 22 | + DatabaseCollation string |
| 23 | + InconsistentCollationColumns []string |
| 24 | + IsCollationCaseSensitive func(s string) bool |
| 25 | + CollationEquals func(a, b string) bool |
| 26 | +} |
| 27 | + |
| 28 | +func findAvailableCollationsMySQL(x *xorm.Engine) (ret container.Set[string], err error) { |
| 29 | + var res []struct { |
| 30 | + Collation string |
| 31 | + } |
| 32 | + if err = x.SQL("SHOW COLLATION WHERE (Collation = 'utf8mb4_bin') OR (Collation LIKE '%\\_as\\_cs%')").Find(&res); err != nil { |
| 33 | + return nil, err |
| 34 | + } |
| 35 | + ret = make(container.Set[string], len(res)) |
| 36 | + for _, r := range res { |
| 37 | + ret.Add(r.Collation) |
| 38 | + } |
| 39 | + return ret, nil |
| 40 | +} |
| 41 | + |
| 42 | +func findAvailableCollationsMSSQL(x *xorm.Engine) (ret container.Set[string], err error) { |
| 43 | + var res []struct { |
| 44 | + Name string |
| 45 | + } |
| 46 | + if err = x.SQL("SELECT * FROM sys.fn_helpcollations() WHERE name LIKE '%\\_CS\\_AS%'").Find(&res); err != nil { |
| 47 | + return nil, err |
| 48 | + } |
| 49 | + ret = make(container.Set[string], len(res)) |
| 50 | + for _, r := range res { |
| 51 | + ret.Add(r.Name) |
| 52 | + } |
| 53 | + return ret, nil |
| 54 | +} |
| 55 | + |
| 56 | +func CheckCollations(x *xorm.Engine) (*CheckCollationsResult, error) { |
| 57 | + dbTables, err := x.DBMetas() |
| 58 | + if err != nil { |
| 59 | + return nil, err |
| 60 | + } |
| 61 | + |
| 62 | + res := &CheckCollationsResult{} |
| 63 | + res.CollationEquals = func(a, b string) bool { return a == b } |
| 64 | + |
| 65 | + var candidateCollations []string |
| 66 | + if x.Dialect().URI().DBType == schemas.MYSQL { |
| 67 | + if _, err = x.SQL("SELECT @@collation_database").Get(&res.DatabaseCollation); err != nil { |
| 68 | + return nil, err |
| 69 | + } |
| 70 | + res.IsCollationCaseSensitive = func(s string) bool { |
| 71 | + return s == "utf8mb4_bin" || strings.HasSuffix(s, "_as_cs") |
| 72 | + } |
| 73 | + candidateCollations = []string{"utf8mb4_0900_as_cs", "uca1400_as_cs", "utf8mb4_bin"} |
| 74 | + res.AvailableCollation, err = findAvailableCollationsMySQL(x) |
| 75 | + if err != nil { |
| 76 | + return nil, err |
| 77 | + } |
| 78 | + res.CollationEquals = func(a, b string) bool { |
| 79 | + // MariaDB adds the "utf8mb4_" prefix, eg: "utf8mb4_uca1400_as_cs", but not the name "uca1400_as_cs" in "SHOW COLLATION" |
| 80 | + // At the moment, it's safe to ignore the database difference, just trim the prefix and compare. It could be fixed easily if there is any problem in the future. |
| 81 | + return a == b || strings.TrimPrefix(a, "utf8mb4_") == strings.TrimPrefix(b, "utf8mb4_") |
| 82 | + } |
| 83 | + } else if x.Dialect().URI().DBType == schemas.MSSQL { |
| 84 | + if _, err = x.SQL("SELECT DATABASEPROPERTYEX(DB_NAME(), 'Collation')").Get(&res.DatabaseCollation); err != nil { |
| 85 | + return nil, err |
| 86 | + } |
| 87 | + res.IsCollationCaseSensitive = func(s string) bool { |
| 88 | + return strings.HasSuffix(s, "_CS_AS") |
| 89 | + } |
| 90 | + candidateCollations = []string{"Latin1_General_CS_AS"} |
| 91 | + res.AvailableCollation, err = findAvailableCollationsMSSQL(x) |
| 92 | + if err != nil { |
| 93 | + return nil, err |
| 94 | + } |
| 95 | + } else { |
| 96 | + return nil, nil |
| 97 | + } |
| 98 | + |
| 99 | + if res.DatabaseCollation == "" { |
| 100 | + return nil, errors.New("unable to get collation for current database") |
| 101 | + } |
| 102 | + |
| 103 | + res.ExpectedCollation = setting.Database.CharsetCollation |
| 104 | + if res.ExpectedCollation == "" { |
| 105 | + for _, collation := range candidateCollations { |
| 106 | + if res.AvailableCollation.Contains(collation) { |
| 107 | + res.ExpectedCollation = collation |
| 108 | + break |
| 109 | + } |
| 110 | + } |
| 111 | + } |
| 112 | + |
| 113 | + if res.ExpectedCollation == "" { |
| 114 | + return nil, errors.New("unable to find a suitable collation for current database") |
| 115 | + } |
| 116 | + |
| 117 | + for _, table := range dbTables { |
| 118 | + for _, col := range table.Columns() { |
| 119 | + if col.Collation != "" && (!res.IsCollationCaseSensitive(col.Collation) || !res.CollationEquals(col.Collation, res.DatabaseCollation)) { |
| 120 | + res.InconsistentCollationColumns = append(res.InconsistentCollationColumns, fmt.Sprintf("%s.%s", table.Name, col.Name)) |
| 121 | + } |
| 122 | + } |
| 123 | + } |
| 124 | + |
| 125 | + return res, nil |
| 126 | +} |
| 127 | + |
| 128 | +func CheckCollationsDefaultEngine() (*CheckCollationsResult, error) { |
| 129 | + return CheckCollations(x) |
| 130 | +} |
| 131 | + |
| 132 | +func alterDatabaseCollation(x *xorm.Engine, checkResult *CheckCollationsResult) error { |
| 133 | + if x.Dialect().URI().DBType == schemas.MYSQL { |
| 134 | + _, err := x.Exec("ALTER DATABASE CHARACTER SET utf8mb4 COLLATE " + checkResult.ExpectedCollation) |
| 135 | + return err |
| 136 | + } else if x.Dialect().URI().DBType == schemas.MSSQL { |
| 137 | + // MSSQL has many limitations on changing database collation, it could fail in many cases |
| 138 | + _, err := x.Exec("ALTER DATABASE CURRENT COLLATE " + checkResult.ExpectedCollation) |
| 139 | + return err |
| 140 | + } |
| 141 | + return errors.New("unsupported database type") |
| 142 | +} |
| 143 | + |
| 144 | +func preprocessDatabaseCollation(x *xorm.Engine) { |
| 145 | + r, err := CheckCollations(x) |
| 146 | + if err != nil { |
| 147 | + log.Error("Failed to check database collation: %v") |
| 148 | + } |
| 149 | + if r == nil { |
| 150 | + return // no check result means the database doesn't need to do such check/process (at the moment ....) |
| 151 | + } |
| 152 | + |
| 153 | + if !r.CollationEquals(r.DatabaseCollation, r.ExpectedCollation) { |
| 154 | + if err = alterDatabaseCollation(x, r); err != nil { |
| 155 | + log.Error("Failed to change database collation to %q: %v", r.ExpectedCollation, err) |
| 156 | + } else { |
| 157 | + if r, err = CheckCollations(x); err != nil { |
| 158 | + log.Fatal("Failed to check database collation again after altering: %v", err) // impossible case |
| 159 | + } |
| 160 | + log.Warn("Current database has been altered to use collation %q", r.DatabaseCollation) |
| 161 | + } |
| 162 | + } |
| 163 | + |
| 164 | + if !r.IsCollationCaseSensitive(r.DatabaseCollation) { |
| 165 | + log.Warn("Current database is using a case-insensitive collation %q, although Gitea could work with it, there might be some rare cases which don't work as expected.", r.DatabaseCollation) |
| 166 | + } |
| 167 | + |
| 168 | + if len(r.InconsistentCollationColumns) > 0 { |
| 169 | + log.Error("There are %d table columns have inconsistent collation, they should use %q. Please go to admin panel Self Check page or refer to Gitea document", len(r.InconsistentCollationColumns), r.DatabaseCollation) |
| 170 | + } |
| 171 | +} |
0 commit comments