Things that weren't obvious

1) IAMVfwCompressDialogs must be used AFTER THE COMPRESSOR IS CONNECTED, otherwise Configure ignores SetState and reinitializes the dialog every time
2) IAMVideoCompression must be created on the OUTPUT PIN of the compressor, not the on compressor itself
3) any size other than 320 x 240 causes first-chance exception in QCAP.DLL (debug only)

more problems: can't seem to set output bit rate, maybe that just doesn't work

IPin    *pComprOut = CDSBmpToAvi::GetPin(ip->pCompr, PINDIR_OUTPUT);
if (pComprOut != NULL) {    // if we got compressor's output pin
    IAMStreamConfig *pStreamCfg = NULL;
    m_hr = pComprOut->QueryInterface(   // get IAMVideoCompression
        IID_IAMStreamConfig, (void **)&pStreamCfg);
    if (pStreamCfg != NULL) {   // do GetFormat after input pin is connected
        AM_MEDIA_TYPE   *pmt;
        m_hr = pStreamCfg->GetFormat(&pmt);
        if (SUCCEEDED(m_hr)) {
            printf("get format ok\n");
            if (pmt->formattype == FORMAT_VideoInfo) {
                VIDEOINFOHEADER *pvh = (VIDEOINFOHEADER *)pmt->pbFormat;
                printf("%d\n", pvh->dwBitRate);
                pvh->dwBitRate = 10000;
                m_hr = pStreamCfg->SetFormat(pmt);
                if (SUCCEEDED(m_hr)) {
                    printf("set format ok\n");
                }
    //          DeleteMediaType(pmt);
            }
        }
    }
}