codec, table: make new collation setting explicit in encoding (#69566)

pkg/ddl/column.go

@@ -44,6 +44,8 @@ import (
 	"github.com/pingcap/tidb/pkg/table"
 	"github.com/pingcap/tidb/pkg/tablecodec"
 	"github.com/pingcap/tidb/pkg/types"
+	"github.com/pingcap/tidb/pkg/util/codec"
+	"github.com/pingcap/tidb/pkg/util/collate"
 	contextutil "github.com/pingcap/tidb/pkg/util/context"
 	"github.com/pingcap/tidb/pkg/util/dbterror"
 	"github.com/pingcap/tidb/pkg/util/intest"
@@ -839,7 +841,8 @@ func (w *updateColumnWorker) getRowRecord(handle kv.Handle, recordKey []byte, ra
 	if w.checksumNeeded {
 		checksum = rowcodec.RawChecksum{Handle: handle}
 	}
-	newRowVal, err := tablecodec.EncodeRow(sysTZ, newRow, newColumnIDs, nil, nil, checksum, rd)
+	enc := codec.NewEncoder(collate.NewCollationEnabled())
+	newRowVal, err := tablecodec.EncodeRow(enc, sysTZ, newRow, newColumnIDs, nil, nil, checksum, rd)
 	err = ec.HandleError(err)
 	if err != nil {
 		return errors.Trace(err)

pkg/ddl/index.go

@@ -74,6 +74,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util"
 	"github.com/pingcap/tidb/pkg/util/backoff"
 	"github.com/pingcap/tidb/pkg/util/chunk"
+	"github.com/pingcap/tidb/pkg/util/collate"
 	"github.com/pingcap/tidb/pkg/util/dbterror"
 	"github.com/pingcap/tidb/pkg/util/generatedexpr"
 	"github.com/pingcap/tidb/pkg/util/intest"
@@ -2092,7 +2093,7 @@ func (w *baseIndexWorker) getIndexRecord(idxInfo *model.IndexInfo, handle kv.Han
 		idxVal[j] = idxColumnVal
 	}
 
-	rsData := tables.TryGetHandleRestoredDataWrapper(w.table.Meta(), nil, w.rowMap, idxInfo)
+	rsData := tables.TryGetHandleRestoredDataWrapper(collate.NewCollationEnabled(), w.table.Meta(), nil, w.rowMap, idxInfo)
 	idxRecord := &indexRecord{handle: handle, key: recordKey, vals: idxVal, rsData: rsData}
 	return idxRecord, nil
 }
@@ -2332,11 +2333,12 @@ func writeChunk(
 	}()
 	needRestoreForIndexes := make([]bool, len(indexes))
 	restore, pkNeedRestore := false, false
+	useNewCollate := collate.NewCollationEnabled()
 	if c.PrimaryKeyInfo != nil && c.TableInfo.IsCommonHandle && c.TableInfo.CommonHandleVersion != 0 {
-		pkNeedRestore = tables.NeedRestoredData(c.PrimaryKeyInfo.Columns, c.TableInfo.Columns)
+		pkNeedRestore = tables.NeedRestoredData(useNewCollate, c.PrimaryKeyInfo.Columns, c.TableInfo.Columns)
 	}
 	for i, index := range indexes {
-		needRestore := pkNeedRestore || tables.NeedRestoredData(index.Meta().Columns, c.TableInfo.Columns)
+		needRestore := pkNeedRestore || tables.NeedRestoredData(useNewCollate, index.Meta().Columns, c.TableInfo.Columns)
 		needRestoreForIndexes[i] = needRestore
 		restore = restore || needRestore
 	}

pkg/domain/domain.go

@@ -542,7 +542,7 @@ const resourceIdleTimeout = 3 * time.Minute // resources in the ResourcePool wil
 
 // NewDomain creates a new domain. Should not create multiple domains for the same store.
 func NewDomain(store kv.Storage, schemaLease time.Duration, statsLease time.Duration, dumpFileGcLease time.Duration, factory pools.Factory) *Domain {
-	return NewDomainWithEtcdClient(store, schemaLease, statsLease, dumpFileGcLease, factory, nil, nil)
+	return NewDomainWithEtcdClient(store, schemaLease, statsLease, dumpFileGcLease, factory, nil, nil, nil)
 }
 
 // NewDomainWithEtcdClient creates a new domain with etcd client. Should not create multiple domains for the same store.
@@ -554,6 +554,7 @@ func NewDomainWithEtcdClient(
 	factory pools.Factory,
 	crossKSSessFactoryGetter func(targetKS string, validator validatorapi.Validator) pools.Factory,
 	etcdClient *clientv3.Client,
+	schemaFilter issyncer.Filter,
 ) *Domain {
 	intest.Assert(schemaLease > 0, "schema lease should be a positive duration")
 	do := &Domain{
@@ -597,6 +598,7 @@ func NewDomainWithEtcdClient(
 		do.schemaLease,
 		do.sysSessionPool,
 		isvalidator.New(do.schemaLease),
+		schemaFilter,
 	)
 	do.initDomainSysVars()
 

pkg/executor/admin.go

@@ -35,6 +35,7 @@ import (
 	"github.com/pingcap/tidb/pkg/types"
 	"github.com/pingcap/tidb/pkg/util"
 	"github.com/pingcap/tidb/pkg/util/chunk"
+	"github.com/pingcap/tidb/pkg/util/collate"
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	"github.com/pingcap/tidb/pkg/util/ranger"
 	"github.com/pingcap/tidb/pkg/util/timeutil"
@@ -363,6 +364,7 @@ func (e *RecoverIndexExec) fetchRecoverRows(ctx context.Context, srcResult dists
 	idxValLen := len(e.index.Meta().Columns)
 	result.scanRowCount = 0
 
+	useNewCollate := collate.NewCollationEnabled()
 	for {
 		err := srcResult.Next(ctx, e.srcChunk)
 		if err != nil {
@@ -389,7 +391,7 @@ func (e *RecoverIndexExec) fetchRecoverRows(ctx context.Context, srcResult dists
 				return nil, err
 			}
 			e.idxValsBufs[result.scanRowCount] = idxVals
-			rsData := tables.TryGetHandleRestoredDataWrapper(e.table.Meta(), plannercore.GetCommonHandleDatum(e.handleCols, row), nil, e.index.Meta())
+			rsData := tables.TryGetHandleRestoredDataWrapper(useNewCollate, e.table.Meta(), plannercore.GetCommonHandleDatum(e.handleCols, row), nil, e.index.Meta())
 			e.recoverRows = append(e.recoverRows, recoverRows{handle: handle, idxVals: idxVals, rsData: rsData, skip: true})
 			result.scanRowCount++
 			result.currentHandle = handle

pkg/executor/test/executor/BUILD.bazel

@@ -47,6 +47,8 @@ go_test(
         "//pkg/testkit/testfailpoint",
         "//pkg/types",
         "//pkg/util",
+        "//pkg/util/codec",
+        "//pkg/util/collate",
         "//pkg/util/dbterror/exeerrors",
         "//pkg/util/dbterror/plannererrors",
         "//pkg/util/mock",

pkg/executor/test/executor/executor_test.go

@@ -67,6 +67,8 @@ import (
 	"github.com/pingcap/tidb/pkg/testkit/testfailpoint"
 	"github.com/pingcap/tidb/pkg/types"
 	"github.com/pingcap/tidb/pkg/util"
+	"github.com/pingcap/tidb/pkg/util/codec"
+	"github.com/pingcap/tidb/pkg/util/collate"
 	"github.com/pingcap/tidb/pkg/util/dbterror/exeerrors"
 	"github.com/pingcap/tidb/pkg/util/dbterror/plannererrors"
 	"github.com/pingcap/tidb/pkg/util/mock"
@@ -251,7 +253,7 @@ func setColValue(t *testing.T, txn kv.Transaction, key kv.Key, v types.Datum) {
 	colIDs := []int64{2, 3}
 	sc := stmtctx.NewStmtCtxWithTimeZone(time.Local)
 	rd := rowcodec.Encoder{Enable: true}
-	value, err := tablecodec.EncodeRow(sc.TimeZone(), row, colIDs, nil, nil, nil, &rd)
+	value, err := tablecodec.EncodeRow(codec.NewEncoder(collate.NewCollationEnabled()), sc.TimeZone(), row, colIDs, nil, nil, nil, &rd)
 	require.NoError(t, err)
 	err = txn.Set(key, value)
 	require.NoError(t, err)

pkg/executor/write.go

@@ -35,6 +35,7 @@ import (
 	"github.com/pingcap/tidb/pkg/tablecodec"
 	"github.com/pingcap/tidb/pkg/types"
 	"github.com/pingcap/tidb/pkg/util/chunk"
+	"github.com/pingcap/tidb/pkg/util/codec"
 	"github.com/pingcap/tidb/pkg/util/collate"
 	"github.com/pingcap/tidb/pkg/util/dbterror/plannererrors"
 	"github.com/pingcap/tidb/pkg/util/memory"
@@ -396,6 +397,7 @@ func addUnchangedKeysForLockByRow(
 				return count, err
 			}
 			unchangedUniqueKey, _, err := tablecodec.GenIndexKey(
+				codec.NewEncoder(collate.NewCollationEnabled()),
 				stmtCtx.TimeZone(),
 				idx.TableMeta(),
 				meta,

pkg/expression/expression.go

@@ -63,6 +63,8 @@ type BuildOptions struct {
 	AllowCastArray bool
 	// TargetFieldType indicates to cast the expression to the target field type if it is not nil
 	TargetFieldType *types.FieldType
+	// UseNewCollate whether to use new collate when building expression.
+	UseNewCollate bool
 }
 
 // BuildOption is a function to apply optional settings
@@ -100,6 +102,14 @@ func WithCastExprTo(targetFt *types.FieldType) BuildOption {
 	}
 }
 
+// WithUseNewCollate fixes the collation mode used while building expressions
+// that must stay consistent with table or index encoding created earlier.
+func WithUseNewCollate(useNewCollate bool) BuildOption {
+	return func(options *BuildOptions) {
+		options.UseNewCollate = useNewCollate
+	}
+}
+
 // BuildSimpleExpr builds a simple expression from an ast node.
 // This function is used to build some "simple" expressions with limited context.
 // The below expressions are not supported:

pkg/infoschema/issyncer/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "issyncer",
     srcs = [
         "deferfn.go",
+        "filter.go",
         "loader.go",
         "mdl_check.go",
         "syncer.go",

pkg/infoschema/issyncer/filter.go

@@ -0,0 +1,37 @@
+// Copyright 2026 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package issyncer
+
+import (
+	"github.com/pingcap/tidb/pkg/infoschema"
+	"github.com/pingcap/tidb/pkg/meta/model"
+)
+
+// Filter allows caller to customize which schema objects should be loaded
+// when syncing the information schema.
+//
+// The semantics of the return values are:
+//   - true:  skip the corresponding operation
+//   - false: continue the default loading logic
+type Filter interface {
+	// SkipLoadDiff returns true when the given schema diff should be ignored.
+	// latestIS is the newest InfoSchema cached by the loader. It can be nil when
+	// the loader hasn't loaded any schema yet.
+	SkipLoadDiff(diff *model.SchemaDiff, latestIS infoschema.InfoSchema) bool
+
+	// SkipLoadSchema returns true when the given DB should not be loaded during
+	// a full schema load.
+	SkipLoadSchema(dbInfo *model.DBInfo) bool
+}

pkg/infoschema/issyncer/loader.go

@@ -85,6 +85,7 @@ type Loader struct {
 	// loading system tables
 	crossKS bool
 	logger  *zap.Logger
+	filter  Filter
 
 	// below fields are set when running background routines
 	// Note: for cross keyspace loader, we don't set below fields as system tables
@@ -98,13 +99,14 @@ type Loader struct {
 	sysExecutorFactory func() (pools.Resource, error)
 }
 
-func newLoader(store kv.Storage, infoCache *infoschema.InfoCache, deferFn *deferFn) *Loader {
+func newLoader(store kv.Storage, infoCache *infoschema.InfoCache, deferFn *deferFn, filter Filter) *Loader {
 	mode := LoadModeAuto
 	return &Loader{
 		mode:      mode,
 		store:     store,
 		infoCache: infoCache,
 		deferFn:   deferFn,
+		filter:    filter,
 		logger:    logutil.BgLogger().With(zap.Stringer("mode", mode)),
 	}
 }
@@ -289,6 +291,16 @@ func (l *Loader) LoadWithTS(startTS uint64, isSnapshot bool) (infoschema.InfoSch
 }
 
 func (l *Loader) skipLoadingDiff(diff *model.SchemaDiff) bool {
+	if l.filter != nil {
+		var latestIS infoschema.InfoSchema
+		if l.infoCache != nil {
+			latestIS = l.infoCache.GetLatest()
+		}
+		if l.filter.SkipLoadDiff(diff, latestIS) {
+			return true
+		}
+	}
+
 	if !l.crossKS {
 		return false
 	}
@@ -412,6 +424,17 @@ func (l *Loader) fetchAllSchemasWithTables(m meta.Reader, schemaCacheSize uint64
 			return nil, errors.New("system database not found")
 		}
 		allSchemas = []*model.DBInfo{dbInfo}
+	} else if l.filter != nil {
+		allSchemas = make([]*model.DBInfo, 0, 6)
+		err := m.IterDatabases(func(dbInfo *model.DBInfo) error {
+			if !l.filter.SkipLoadSchema(dbInfo) {
+				allSchemas = append(allSchemas, dbInfo)
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
 	} else {
 		allSchemas, err = m.ListDatabases()
 		if err != nil {

pkg/infoschema/issyncer/loader_test.go

@@ -33,7 +33,7 @@ import (
 func TestLoadFromTS(t *testing.T) {
 	store, err := mockstore.NewMockStore()
 	require.NoError(t, err)
-	l := newLoader(store, infoschema.NewCache(nil, 1), nil)
+	l := newLoader(store, infoschema.NewCache(nil, 1), nil, nil)
 	ver, err := store.CurrentVersion(tidbkv.GlobalTxnScope)
 	require.NoError(t, err)
 	is, hitCache, oldSchemaVersion, changes, err := l.LoadWithTS(ver.Ver, false)
@@ -70,7 +70,7 @@ func TestLoadFromTS(t *testing.T) {
 	}))
 	ver, err = store.CurrentVersion(tidbkv.GlobalTxnScope)
 	require.NoError(t, err)
-	l = newLoader(store, infoschema.NewCache(nil, 1), nil)
+	l = newLoader(store, infoschema.NewCache(nil, 1), nil, nil)
 	is, hitCache, oldSchemaVersion, changes, err = l.LoadWithTS(ver.Ver, false)
 	require.NoError(t, err)
 	allSchemas = is.AllSchemas()
@@ -217,7 +217,7 @@ func (testStoreWithKS) GetKeyspace() string {
 }
 
 func TestLoaderSkipLoadingDiff(t *testing.T) {
-	syncer := New(nil, nil, 0, nil, nil)
+	syncer := New(nil, nil, 0, nil, nil, nil)
 	require.False(t, syncer.loader.skipLoadingDiff(&model.SchemaDiff{}))
 	require.False(t, syncer.loader.skipLoadingDiff(&model.SchemaDiff{TableID: 100}))
 	require.False(t, syncer.loader.skipLoadingDiff(&model.SchemaDiff{OldTableID: 100}))

pkg/infoschema/issyncer/syncer.go

@@ -91,9 +91,10 @@ func New(
 	schemaLease time.Duration,
 	sysSessionPool util.DestroyableSessionPool,
 	isValidator validatorapi.Validator,
+	filter Filter,
 ) *Syncer {
 	s := newSyncer(store, logutil.BgLogger(), schemaLease, sysSessionPool, isValidator)
-	s.loader = newLoader(store, infoCache, &s.deferFn)
+	s.loader = newLoader(store, infoCache, &s.deferFn, filter)
 	return s
 }
 

pkg/infoschema/issyncer/syncer_test.go

@@ -22,7 +22,7 @@ import (
 )
 
 func TestSyncerSkipMDLCheck(t *testing.T) {
-	syncer := New(nil, nil, 0, nil, nil)
+	syncer := New(nil, nil, 0, nil, nil, nil)
 	require.False(t, syncer.skipMDLCheck(map[int64]struct{}{}))
 	require.False(t, syncer.skipMDLCheck(map[int64]struct{}{123: {}}))
 	require.False(t, syncer.skipMDLCheck(map[int64]struct{}{123: {}, 456: {}}))

pkg/meta/meta.go

@@ -1045,6 +1045,19 @@ func (m *Mutator) UpdateTable(dbID int64, tableInfo *model.TableInfo) error {
 	return errors.Trace(err)
 }
 
+// IterDatabases iterates all the Databases at once, stop iterate when fn returns an error.
+func (m *Mutator) IterDatabases(fn func(info *model.DBInfo) error) error {
+	err := m.txn.HGetIter(mDBs, func(r structure.HashPair) error {
+		dbInfo := &model.DBInfo{}
+		err := json.Unmarshal(r.Value, dbInfo)
+		if err != nil {
+			return errors.Trace(err)
+		}
+		return fn(dbInfo)
+	})
+	return errors.Trace(err)
+}
+
 // IterTables iterates all the table at once, in order to avoid oom.
 func (m *Mutator) IterTables(dbID int64, fn func(info *model.TableInfo) error) error {
 	dbKey := m.dbKey(dbID)

pkg/meta/reader.go

@@ -29,6 +29,7 @@ type Reader interface {
 	GetTable(dbID int64, tableID int64) (*model.TableInfo, error)
 	ListTables(ctx context.Context, dbID int64) ([]*model.TableInfo, error)
 	ListSimpleTables(dbID int64) ([]*model.TableNameInfo, error)
+	IterDatabases(func(info *model.DBInfo) error) error
 	IterTables(dbID int64, fn func(info *model.TableInfo) error) error
 	GetAutoIDAccessors(dbID, tableID int64) AutoIDAccessors
 	GetAllNameToIDAndTheMustLoadedTableInfo(dbID int64) (map[string]int64, []*model.TableInfo, error)