Fg3PhotofitUtil.cpp

//
// Copyright (c) 2023 Singular Inversions Inc. (facegen.com)
// Created: May 29, 2003.
// Authors: John Leung, Andrew Beatty
//
// Demonstrates the use of the PhotoFit DLL to create FG data from an image.
//
 
#include "stdafx.h"
 
#include "Fg3PhotofitUtil.hpp"
#include "FgFile.hpp"
#include "FgFileSystem.hpp"
#include "FgGuiApi.hpp"
 
using namespace std;
 
namespace Fg {
 
#define FMOSYNTH_TOP_SL_SIZE 512
 
Fg3InitPoints       getSetLandmarks(String8 const & photoFile,ImgRgba8 const & photo)
{
    String8             lmsFile = pathToDirBase(photoFile)+".lms.txt";
    Svec<FutVect2IC>    lmPoss;
    if (fileExists(lmsFile)) {
        FaceLmPos2Fs        flmPoss = toFaceLmPoss(loadLandmarks(lmsFile));
        FaceLms             flms = mapMember(flmPoss,&FaceLmPos2F::lm);
        for (FaceView fv : getAllE<FaceView>()) {
            FaceLms             viewLms = mapMember(getV3LmAtts(fv),&FaceLmAtt::type);
            if (containsAll(flms,viewLms)) {        // this must be the view as we have all LMs for it
                for (FaceLm lm : viewLms) {
                    FaceLmPos2F     lmPos = findFirst(flmPoss,lm);
                    lmPoss.emplace_back(lmPos.pos[0],lmPos.pos[1]);
                }
                break;
            }
        }
    }
    Fg3InitPoints       landmarks;
    landmarks.set(lmPoss);
    if (landmarks.empty()) {
        fgout << fgnl <<
            "Select the bootstrap points by left-clicking on the displayed photograph window.\n"
            "Follow the guidlines shown in the Modeller or in the SDK documentation.\n"
            "When finished, close the photograph window. If the window re-opens with the same\n"
            "photograph, you have not selected the correct number of points.\n"
            "Right-click and drag up/down to scale the image.\n";
        bool                done = false;
        while (!done) {
            String8             store = getDirUserAppDataLocal({"FaceGen","fg3pf"});
            IPT<ImgRgba8>       imgN = makeIPT(photo);
            IPT<NameVec2Fs>     ptsIucsN = makeIPT(NameVec2Fs{});
            auto                leftClickFn = [ptsIucsN](Vec2F iucs,Vec2UI){ptsIucsN.ref().emplace_back("",iucs); };
            GuiPtr              guiImg = guiImageCtrls(imgN,ptsIucsN,false,leftClickFn).win;
            try {
                guiStartImpl(makeIPT<String8>("fg3pf"),guiImg,store);
            }
            catch (FgExceptionNotImplemented & e) {
                e.contexts.emplace_back("No GUI, use a .lms.txt file to specify the landmark points.");
                throw e;
            }
            AxAffine2F          iucsToPacs = cIucsToPacs<float>(photo.dims());
            vector<FutVect2IC>  ptsIrcs;
            for (NameVec2F iucs : ptsIucsN.val()) {
                Vec2F               pacs = iucsToPacs * iucs.vec;
                FutVect2IC          p;
                p.x1 = std::floor(pacs[0]);
                p.x2 = std::floor(pacs[1]);
                ptsIrcs.push_back(p);
            }
            if (ptsIrcs.empty())
                fgThrow("User abort");
            done = landmarks.set(ptsIrcs);
        }
        saveLandmarks(landmarks.asNameVecs(),lmsFile);
    }
    FutVect2FC          lo,hi;
    landmarks.getCropVals(lo,hi);
    if ((hi.x2 - lo.x2) < (float)FMOSYNTH_TOP_SL_SIZE / 8.0f)
        fgout << fgnl << "Warning: " << photoFile << " is too small for fitting";
    else if ((hi.x2 - lo.x2) < (float)FMOSYNTH_TOP_SL_SIZE / 2.0f)
        fgout << fgnl << "Warning: " << photoFile << " is too small for optimal fitting.";
    if ((hi.x2 - lo.x2) < (float)FMOSYNTH_TOP_SL_SIZE)
        fgout << fgnl << "Warning: " << photoFile << " is too small for optimal detail extraction.";
    return landmarks;
}
 
void                copyPoints(const Fg3InitPoints & src,FgppUserDataS & dst)
{
    uint                numPts = 0;
    for (int jj=0; jj<LM3_SIZE; jj++) {
        if (src.m_valid[jj]) {
            dst.pointX[numPts] = uint(src.m_pos[jj].x1 + 0.5f);
            dst.pointY[numPts] = uint(src.m_pos[jj].x2 + 0.5f);
            numPts++;
        }
    }
    dst.numPoints = numPts;
}
 
FgppUserDataS       cFgppUserData(ImgRgba8 const & photo,Fg3InitPoints const & landmarks)
{
    FgppUserDataS       data;
    data.width = photo.width();
    data.height = photo.height();
    data.widthStep = photo.width();
    data.imageData = photo.dataPtr();
    copyPoints(landmarks,data);
    return data;
}
 
FaceLmAtts const &   getV3LmAtts(FaceView view)
{
    static Arr<FaceLmAtts,3> ret {
        {   // front
            {FaceLm::pupilCentreL,"EYE_LEFT_CENTRE"},
            {FaceLm::pupilCentreR,"EYE_RIGHT_CENTRE"},
            {FaceLm::cheekboneL,"CHEEKBONE_LEFT"},
            {FaceLm::cheekboneR,"CHEEKBONE_RIGHT"},
            {FaceLm::alarOuterL,"NARE_LEFT"},
            {FaceLm::alarOuterR,"NARE_RIGHT"},
            {FaceLm::mouthCornerL,"MOUTH_LEFT"},
            {FaceLm::mouthCornerR,"MOUTH_RIGHT"},
            {FaceLm::jawOuterL,"JAW_OUTER_LEFT"},
            {FaceLm::jawOuterR,"JAW_OUTER_RIGHT"},
            {FaceLm::chinBottom,"CHIN_LOWER"},
        },
        {   // left
            {FaceLm::exocanthionL,"EYE_LEFT_OUTER"},
            {FaceLm::sellion,"SELLION"},
            {FaceLm::noseBridgeMiddle,"NOSE_BRIDGE"},
            {FaceLm::noseTip,"NOSE_TIP"},
            {FaceLm::subNasale,"NOSE_BASE"},
            {FaceLm::lipUpperFront,"LIP_TIP_UPPER"},
            {FaceLm::lipLowerFront,"LIP_TIP_LOWER"},
            {FaceLm::chinFront,"CHIN"},
            {FaceLm::jawBottom,"THROAT_TOP"},
        },
        {   // right
            {FaceLm::exocanthionR,"EYE_RIGHT_OUTER"},
            {FaceLm::sellion,"SELLION"},
            {FaceLm::noseBridgeMiddle,"NOSE_BRIDGE"},
            {FaceLm::noseTip,"NOSE_TIP"},
            {FaceLm::subNasale,"NOSE_BASE"},
            {FaceLm::lipUpperFront,"LIP_TIP_UPPER"},
            {FaceLm::lipLowerFront,"LIP_TIP_LOWER"},
            {FaceLm::chinFront,"CHIN"},
            {FaceLm::jawBottom,"THROAT_TOP"},
        },
    };
    return ret[size_t(view)];
}
 
ViewLmPoss           parseLmsV3(Vec2Fs const & pacs)
{
    ViewLmPoss           ret;
    auto                ltX = [](Vec2F l,Vec2F r){return l[0] < r[0]; };
    auto                ltY = [](Vec2F l,Vec2F r){return l[1] < r[1]; };
    Vec2Fs              ptsY = sortAll(pacs,ltY);
    if (ptsY.size() == 9) {          // profile
        bool                right = ptsY[8][0] < ptsY[7][0];        // compare chin front & jaw bottom X
        ret.lms.emplace_back(FaceLm::lipUpperFront,ptsY[5]);
        ret.lms.emplace_back(FaceLm::lipLowerFront,ptsY[6]);
        ret.lms.emplace_back(FaceLm::chinFront,ptsY[7]);
        ret.lms.emplace_back(FaceLm::jawBottom,ptsY[8]);
        Vec2Fs              sv05X = sortAll(cSubvec(ptsY,0,5),ltX);
        if (right) {
            ret.view = FaceView::right;
            ret.lms.emplace_back(FaceLm::exocanthionR,sv05X[0]);
            ret.lms.emplace_back(FaceLm::noseTip,sv05X[4]);
        }
        else {
            ret.view = FaceView::left;
            ret.lms.emplace_back(FaceLm::noseTip,sv05X[0]);
            ret.lms.emplace_back(FaceLm::exocanthionL,sv05X[4]);
        }
        Vec2Fs              sv05X13Y = sortAll(cSubvec(sv05X,1,3),ltY);
        ret.lms.emplace_back(FaceLm::sellion,sv05X13Y[0]);
        ret.lms.emplace_back(FaceLm::noseBridgeMiddle,sv05X13Y[1]);
        ret.lms.emplace_back(FaceLm::subNasale,sv05X13Y[2]);
    }
    else if (ptsY.size() == 11) {    // frontal
        ret.view = FaceView::front;
        ret.lms.emplace_back(FaceLm::chinBottom,ptsY.back());
        Vec2Fs              sv64X = sortAll(cSubvec(ptsY,6,4),ltX);
        ret.lms.emplace_back(FaceLm::jawOuterR,sv64X[0]);
        ret.lms.emplace_back(FaceLm::mouthCornerR,sv64X[1]);
        ret.lms.emplace_back(FaceLm::mouthCornerL,sv64X[2]);
        ret.lms.emplace_back(FaceLm::jawOuterL,sv64X[3]);
        Vec2Fs              sv06X = sortAll(cSubvec(ptsY,0,6),ltX);
        ret.lms.emplace_back(FaceLm::cheekboneR,sv06X[0]);
        ret.lms.emplace_back(FaceLm::cheekboneL,sv06X[5]);
        Vec2Fs              sv06X14Y = sortAll(cSubvec(sv06X,1,4),ltY);
        Vec2Fs              sv06X14Y22X = sortAll(cSubvec(sv06X14Y,2,2),ltX);
        ret.lms.emplace_back(FaceLm::alarOuterR,sv06X14Y22X[0]);
        ret.lms.emplace_back(FaceLm::alarOuterL,sv06X14Y22X[1]);
        Vec2Fs              sv06X14Y02X = sortAll(cSubvec(sv06X14Y,0,2),ltX);
        ret.lms.emplace_back(FaceLm::pupilCentreR,sv06X14Y02X[0]);
        ret.lms.emplace_back(FaceLm::pupilCentreL,sv06X14Y02X[1]);
    }
    else
        fgThrow("Not a valid number of landmarks for V3");
    return ret;
}
 
}