Still curious if anyone has an elegant solution for this. This has broken all my interfaces. Since also the ColorSlider is affected. (left 13.1 right 13.2)
Table[ColorSlider[Red, ImageSize -> {w, h}], {h, 5, 55, 5}, {w, 50, 
   250, 50}] // Grid
Grid[Table[
  ColorSetter[Red, ImageSize -> {w, h}, BaseStyle -> {Blue}], {h, 5, 
   45, 5}, {w, 5, 45, 5}]]

As a workaround i wrote some code that makes a button that acts like a ColorSetter
col1 = Red;
Button[
 Dynamic[Graphics[{col1, Rectangle[]}, ImageSize -> {20, 20}]],
 new = SystemDialogInput["Color", col1];
 col1 = If[new === $Canceled, col1, new];,
 Background -> White, Method -> "Queued", FrameMargins -> 0, 
 Appearance -> "Frameless"]
Not The most elegant solution but works in Manipulate just like a ColorSetter.
Manipulate[
 {col1, col2},
 Control[{
   {col1, Blue, "color 1"}, Button[
     Dynamic[Graphics[{col1, Rectangle[]}, ImageSize -> {20, 20}]],
     new = SystemDialogInput["Color", #];
     col1 = If[new === $Canceled, col1, new];
     , Background -> White, Method -> "Queued", FrameMargins -> 0, 
     Appearance -> "Frameless"] &
   }],
 Control[{
   {col2, Red, "color 2"},
   ColorSetter[#, ImageSize -> {50, 50}] &
   }]
 ]
