By lack of better, I am using the following combination of Overlay and SetAplhaChannel to mask images with square or round masks (illustrated with a Manipulate):
Manipulate[Module[{im, sz, mask},
im = ImageResize[ExampleData[{"TestImage", "Lena"}], 350];
sz = ImageMeasurements[im, "Dimensions"];
mask = ImageResize[
ColorNegate@
Blur[Image[Graphics[RegularPolygon[pos, r, n], PlotRange -> 1]],
blur], Most@sz];
Overlay[{Switch[bg, "black", mask, "white", Nothing],
SetAlphaChannel[im, mask]}]],
Grid[{
{Control[{{r, .95, "size"}, .5, 1.25, ImageSize -> Small}],
Control[{{blur, 0}, 0, 50, ImageSize -> Small}]},
{Control[{{n, 4, "shape"}, {3 -> "triangle", 4 -> "square",
36 -> "round"}}],
Control[{{pos, {0, 0}, "position"}, {-.5, -.5}, {.5, .5},
Slider2D, ImageSize -> 70}]},
{Control[{{bg, "black", "color"}, {"black", "white"}}],
SpanFromAbove}}, Alignment -> {Left}, ItemSize -> 22]]
here are some results:
