diff --git a/packages/edit-site/src/components/sidebar-navigation-screen-pattern/use-pattern-details.js b/packages/edit-site/src/components/sidebar-navigation-screen-pattern/use-pattern-details.js
index fb6be8c7fa2f94..674d95e02a13b1 100644
--- a/packages/edit-site/src/components/sidebar-navigation-screen-pattern/use-pattern-details.js
+++ b/packages/edit-site/src/components/sidebar-navigation-screen-pattern/use-pattern-details.js
@@ -35,18 +35,13 @@ export default function usePatternDetails( postType, postId ) {
 			select( editorStore ).__experimentalGetDefaultTemplatePartAreas(),
 		[]
 	);
-	const { currentTheme, patternCategories } = useSelect( ( select ) => {
+	const { currentTheme, userPatternCategories } = useSelect( ( select ) => {
 		const { getCurrentTheme, getUserPatternCategories } =
 			select( coreStore );
-		const userPatternCategories = getUserPatternCategories();
-		const categories = new Map();
-		userPatternCategories.forEach( ( userCategory ) =>
-			categories.set( userCategory.id, userCategory )
-		);
 
 		return {
 			currentTheme: getCurrentTheme(),
-			patternCategories: categories,
+			userPatternCategories: getUserPatternCategories(),
 		};
 	}, [] );
 
@@ -101,6 +96,11 @@ export default function usePatternDetails( postType, postId ) {
 			} );
 		}
 		if ( record.wp_pattern_category?.length > 0 ) {
+			const patternCategories = new Map();
+			userPatternCategories.forEach( ( userCategory ) =>
+				patternCategories.set( userCategory.id, userCategory )
+			);
+
 			const categories = record.wp_pattern_category
 				.filter( ( category ) => patternCategories.get( category ) )
 				.map( ( category ) => patternCategories.get( category ).label );