One way is to create a custom ChartElementFunction that takes the extra dimension of data as metadata. Here's an example in 3D that should illustrate the idea (and should be easily modifiable for your purposes.)
Brett
In[1]:= ChartElementData[BarChart3D]
Out[1]= {"Cone", "Cube", "Cylinder", "DoubleProfileCube", "FadingCube", "GradientScaleCube", "ProfileCube", "SegmentScaleCube", "SquareWaveCube", "TriangleWaveCube"}
ClearAll[StackedBar3D];
StackedBar3D[scheme_, shape_String: "ProfileCube"][{{x0_, x1_}, {y0_, y1_}, {z0_, z1_}},
val_, meta_]:=Block[{colorlist, n, offsets, heights},
heights = Flatten[meta];
offsets = Prepend[Accumulate[heights], 0];
colorlist = Switch[scheme,_Integer, ColorData[scheme, "ColorList"],_List, scheme,_String, ColorData[scheme, #] & /@ Range[0, 1,1/10]];
n = Length[colorlist];
Table[{colorlist[[Mod[i, n, 1]]],
Tooltip[ChartElementData[shape][{{x0, x1}, {y0, y1}, {offsets[[i]], offsets[[i + 1]]}}, val, meta], heights[i]]}, {i, Length[offsets] - 1}]
]
d1 = RandomReal[1, {4, 4}];
d2 = RandomReal[1, {4, 4}];
d3 = RandomReal[1, {4, 4}];
d4 = RandomReal[1, {4, 4}];
data = MapThread[Plus[##] -> {##} &, {d1, d2, d3, d4}, 2];
BarChart3D[data, ChartElementFunction -> StackedBar3D["Rainbow"], ChartLayout -> "Grid"]
