manipulate.m 724 bytes
%order=['4' - QPSK, '16' - 16QAM, '64' - 64QAM]
%inp - input array
function out = manipulate(inp,order)
out=[];
switch order
    case 4
        check=[-1 1];
        for i=1:2:length(inp)
            out=[out check(inp(i)+1)+j*check(inp(i+1)+1)];
        end
        out=out./sqrt(2);
    case 16
        check=[-3 -1 3 1];
        for i=1:4:length(inp)
            out=[out check(bin2dec(num2str(inp(i:i+1)))+1)+j*check(bin2dec(num2str(inp(i+2:i+3)))+1)];
        end
        out=out./sqrt(10);
    case 64
        check=[-7 -5 -1 -3 7 5 1 3];
        for i=1:6:length(inp)
            out=[out check(bin2dec(num2str(inp(i:i+2)))+1)+j*check(bin2dec(num2str(inp(i+3:i+5)))+1)];
        end
        out=out./sqrt(42);
end
end