In[1]:= (* First polygon is a square centered at the origin,rotated \
by Pi/7, with distances from the center to the vertices equal to 1*)
sqr = Polygon[ RotationMatrix[\[Pi]/7] . # & /@ CirclePoints[4] // N];

In[2]:=  (* Second polygon is a triangle with distances from the \
center to the vertices equal to 0.75, centered at {-2, 1.5} and \
rotated by Pi/5 *)
tri = Polygon[{-2, 1.5} + RotationMatrix[\[Pi]/5] . # & /@ 
    CirclePoints[0.75, 3]];

In[3]:= (* Constraints for a point inside the square*)
sqrCons = List @@ RegionWithin[sqr, Point[{x1, y1}]]

Out[3]= {1. x1 - 2.07652 y1 <= 1.62971, -1.27416 x1 - 0.613604 y1 <= 
  1., 1. x1 + 0.481575 y1 <= 0.784829, -1. x1 + 2.07652 y1 <= 1.62971}

In[4]:= (* Constraints for a point inside the triangle*)
triCons = List @@ RegionWithin[tri, Point[{x2, y2}]]

Out[4]= {1. x2 - 1.37638 y2 <= -3.42658, -1.29192 x2 - 0.135786 y2 <= 
  2.8673, 1. x2 + 2.24604 y2 <= 2.29103}

In[5]:= (* Minimize the distance between the polygons using Quadratic \
Optimization *)
soln = QuadraticOptimization[(x1 - x2)^2 + (y1 - y2)^2, 
  Join[sqrCons, triCons], {x1, x2, y1, y2}]

Out[5]= {x1 -> -0.943883, x2 -> -1.64458, y1 -> 0.330279, y2 -> 1.2947}

In[6]:= (* Calculate the distance *)
dist = Sqrt[(x1 - x2)^2 + (y1 - y2)^2 /. soln]

Out[6]= 1.19209

In[7]:= (* Show the result *)
Show[
 RegionPlot[And @@ sqrCons, {x1, -3, 3}, {y1, -3, 3}, 
  ImageSize -> Small],
 RegionPlot[And @@ triCons, {x2, -3, 3}, {y2, -3, 3}, 
  ImageSize -> Small],
 Graphics @ Line[{{x1, y1}, {x2, y2}} /. soln]
 ]