In some cases, evaluation of expressions containing matrices can be sped up by wrapping the expression in Expand. Here's an example of a 5 fold gain in speed for an expression containing 50x50 matrices:
In[12]:= n = 50;
In[13]:= SeedRandom[0]; Dimensions[a = RandomReal[{-1, 1}, {n, n}]]
Out[13]= {50, 50}
In[14]:= SeedRandom[0]; Dimensions[b = RandomReal[{-1, 1}, n]]
Out[14]= {50}
In[15]:= v = Array[x, n];
In[16]:= AbsoluteTiming @ NMaximize[-1/4 v.a.Transpose[a].v - b.v, v]
Out[16]= {7.294668, {51.9929, {x[1] -> -8.79, x[2] -> -5.14004,
x[3] -> 4.77431, x[4] -> -33.5074, x[5] -> -17.8675,
x[6] -> -41.973, x[7] -> -14.8, x[8] -> 7.17246, x[9] -> 40.5797,
x[10] -> -1.30352, x[11] -> 15.2247, x[12] -> 24.7157,
x[13] -> 20.5358, x[14] -> -37.6259, x[15] -> -23.1615,
x[16] -> 5.04749, x[17] -> 0.716219, x[18] -> -9.14941,
x[19] -> 18.7447, x[20] -> 14.4649, x[21] -> 9.51407,
x[22] -> -23.4491, x[23] -> 1.5991, x[24] -> 6.41455,
x[25] -> 2.53314, x[26] -> 17.8861, x[27] -> -13.0104,
x[28] -> 10.2575, x[29] -> -16.721, x[30] -> 15.549,
x[31] -> -1.68674, x[32] -> 12.4654, x[33] -> 24.0295,
x[34] -> -17.7568, x[35] -> 9.24454, x[36] -> -11.9835,
x[37] -> -5.83835, x[38] -> 0.33071, x[39] -> 16.9103,
x[40] -> -30.5781, x[41] -> -18.2771, x[42] -> -29.8421,
x[43] -> -7.07211, x[44] -> -14.7736, x[45] -> -14.2617,
x[46] -> 39.7569, x[47] -> 48.0993, x[48] -> -3.96191,
x[49] -> 9.70208, x[50] -> -21.7266}}}
In[17]:= AbsoluteTiming @
NMaximize[Expand[-1/4 v.a.Transpose[a].v - b.v], v]
Out[17]= {1.386620, {51.9929, {x[1] -> -8.79, x[2] -> -5.14004,
x[3] -> 4.77431, x[4] -> -33.5074, x[5] -> -17.8675,
x[6] -> -41.973, x[7] -> -14.8, x[8] -> 7.17246, x[9] -> 40.5797,
x[10] -> -1.30352, x[11] -> 15.2247, x[12] -> 24.7157,
x[13] -> 20.5358, x[14] -> -37.6259, x[15] -> -23.1615,
x[16] -> 5.04749, x[17] -> 0.716219, x[18] -> -9.14941,
x[19] -> 18.7447, x[20] -> 14.4649, x[21] -> 9.51407,
x[22] -> -23.4491, x[23] -> 1.5991, x[24] -> 6.41455,
x[25] -> 2.53314, x[26] -> 17.8861, x[27] -> -13.0104,
x[28] -> 10.2575, x[29] -> -16.721, x[30] -> 15.549,
x[31] -> -1.68674, x[32] -> 12.4654, x[33] -> 24.0295,
x[34] -> -17.7568, x[35] -> 9.24454, x[36] -> -11.9835,
x[37] -> -5.83835, x[38] -> 0.33071, x[39] -> 16.9103,
x[40] -> -30.5781, x[41] -> -18.2771, x[42] -> -29.8421,
x[43] -> -7.07211, x[44] -> -14.7736, x[45] -> -14.2617,
x[46] -> 39.7569, x[47] -> 48.0993, x[48] -> -3.96191,
x[49] -> 9.70208, x[50] -> -21.7266}}}