init

.gitattributes

@@ -32,3 +32,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/imgs/cityscape.jpg filter=lfs diff=lfs merge=lfs -text
+assets/imgs/i4.png filter=lfs diff=lfs merge=lfs -text
+models/cvpr2023.pth filter=lfs diff=lfs merge=lfs -text
+models/paramnet_gsv_rpfpp.pth filter=lfs diff=lfs merge=lfs -text
+models/paramnet_gsv_rpf.pth filter=lfs diff=lfs merge=lfs -text
+assets/imgs/907px-Vincent_van_Gogh_-_De_slaapkamer_-_Google_Art_Project.jpg filter=lfs diff=lfs merge=lfs -text
+assets/imgs/AdobeStock_286429091.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/imgs/AdobeStock_331358641.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/imgs/ filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,192 @@
+
+import os
+token = os.environ["GITHUB_TOKEN"]
+os.system(f"pip install git+https://jinlinyi:{token}@github.com:jinlinyi/PerspectiveFields.git@dev")
+
+import gradio as gr
+import cv2
+import copy
+import torch
+from PIL import Image, ImageDraw
+from glob import glob
+import numpy as np
+import os.path as osp
+from detectron2.config import get_cfg
+from detectron2.data.detection_utils import read_image
+from perspective2d.utils.predictor import VisualizationDemo
+import perspective2d.modeling  # noqa
+from perspective2d.config import get_perspective2d_cfg_defaults
+from perspective2d.utils import draw_from_r_p_f_cx_cy
+
+
+
+
+title = "Perspective Fields Demo"
+
+description = """
+<p style="text-align: center">
+    <a href="https://jinlinyi.github.io/PerspectiveFields/" target="_blank">Project Page</a> | 
+    <a href="https://arxiv.org/abs/2212.03239" target="_blank">Paper</a> | 
+    <a href="https://github.com/jinlinyi/PerspectiveFields" target="_blank">Code</a> | 
+    <a href="https://www.youtube.com/watch?v=sN5B_ZvMva8&themeRefresh=1" target="_blank">Video</a>
+</p>
+<h2>Gradio Demo</h2>
+<p>Try our Gradio demo for Perspective Fields for single image camera calibration. You can click on one of the provided examples or upload your own image.</p>
+<h3>Available Models:</h3>
+<ol>
+    <li><strong>PersNet-360Cities:</strong> PerspectiveNet trained on the 360Cities dataset. This model predicts perspective fields and is designed to be robust and generalize well to both indoor and outdoor images.</li>
+    <li><strong>PersNet_Paramnet-GSV-uncentered:</strong> A combination of PerspectiveNet and ParamNet trained on the Google Street View (GSV) dataset. This model predicts camera Roll, Pitch, and Field of View (FoV), as well as the Principal Point location.</li>
+    <li><strong>PersNet_Paramnet-GSV-centered:</strong> PerspectiveNet+ParamNet trained on the GSV dataset. This model assumes the principal point is at the center of the image and predicts camera Roll, Pitch, and FoV.</li>
+</ol>
+"""
+
+
+article = """
+<p style='text-align: center'><a href='https://arxiv.org/abs/2212.03239' target='_blank'>Perspective Fields for Single Image Camera Calibrations</a> | <a href='https://github.com/jinlinyi/PerspectiveFields' target='_blank'>Github Repo</a></p>
+"""
+
+def setup_cfg(args):
+    cfgs = []
+    configs = args['config_file'].split('#')
+    weights_id = args['opts'].index('MODEL.WEIGHTS') + 1
+    weights = args['opts'][weights_id].split('#')
+    for i, conf in enumerate(configs):
+        if len(conf) != 0:
+            tmp_opts = copy.deepcopy(args['opts'])
+            tmp_opts[weights_id] = weights[i]
+            cfg = get_cfg()
+            get_perspective2d_cfg_defaults(cfg)
+            cfg.merge_from_file(conf)
+            cfg.merge_from_list(tmp_opts)
+            cfg.freeze()
+            cfgs.append(cfg)
+    return cfgs
+
+def resize_fix_aspect_ratio(img, field, target_width=None, target_height=None):
+    height = img.shape[0]
+    width = img.shape[1]
+    if target_height is None:
+        factor = target_width / width
+    elif target_width is None:
+        factor = target_height / height
+    else:
+        factor = max(target_width / width, target_height / height)
+    if factor == target_width / width:
+        target_height = int(height * factor)
+    else:
+        target_width = int(width * factor)
+        
+    img = cv2.resize(img, (target_width, target_height))
+    for key in field:
+        if key not in ['up', 'lati']:
+            continue
+        tmp = field[key].numpy()
+        transpose = len(tmp.shape) == 3
+        if transpose:
+            tmp = tmp.transpose(1,2,0)
+        tmp = cv2.resize(tmp, (target_width, target_height))
+        if transpose:
+            tmp = tmp.transpose(2,0,1)
+        field[key] = torch.tensor(tmp)
+    return img, field
+
+
+def inference(img, model_type):
+    perspective_cfg_list = setup_cfg(model_zoo[model_type])
+    demo = VisualizationDemo(cfg_list=perspective_cfg_list)
+
+    # img = read_image(image_path, format="BGR")
+    img = img[..., ::-1] # rgb->bgr
+    pred = demo.run_on_image(img)
+    field = {
+        'up': pred['pred_gravity_original'].cpu().detach(),
+        'lati': pred['pred_latitude_original'].cpu().detach(),
+    }
+    img, field = resize_fix_aspect_ratio(img, field, 640)
+    if not model_zoo[model_type]['param']:
+        pred_vis = demo.draw(
+            image=img,
+            latimap=field['lati'],
+            gravity=field['up'],
+            latimap_format=pred['pred_latitude_original_mode'],
+        ).get_image()
+        param = "Not Implemented"
+    else:
+        if 'pred_general_vfov' not in pred.keys():
+            pred['pred_general_vfov'] = pred['pred_vfov']
+        if 'pred_rel_cx' not in pred.keys():
+            pred['pred_rel_cx'] = torch.FloatTensor([0])
+        if 'pred_rel_cy' not in pred.keys():
+            pred['pred_rel_cy'] = torch.FloatTensor([0])
+            
+        r_p_f_rad = np.radians(
+            [
+                pred['pred_roll'].cpu().item(),
+                pred['pred_pitch'].cpu().item(),
+                pred['pred_general_vfov'].cpu().item(),
+            ]
+        )
+        cx_cy = [
+            pred['pred_rel_cx'].cpu().item(),
+            pred['pred_rel_cy'].cpu().item(),
+        ]
+        param = f"roll {pred['pred_roll'].cpu().item() :.2f}\npitch {pred['pred_pitch'].cpu().item() :.2f}\nfov {pred['pred_general_vfov'].cpu().item() :.2f}\n"
+        param += f"principal point {pred['pred_rel_cx'].cpu().item() :.2f} {pred['pred_rel_cy'].cpu().item() :.2f}"
+        pred_vis = draw_from_r_p_f_cx_cy(
+            img[:,:,::-1], 
+            *r_p_f_rad,
+            *cx_cy,
+            'rad',
+            up_color=(0,1,0),
+        )
+    return Image.fromarray(pred_vis), param
+
+examples = []
+for img_name in glob('assets/imgs/*.*g'):
+    examples.append([img_name])
+print(examples)
+
+model_zoo = {
+    'PersNet-360Cities': {
+        'weights': ['https://www.dropbox.com/s/czqrepqe7x70b7y/cvpr2023.pth'],
+        'opts': ['MODEL.WEIGHTS', 'models/cvpr2023.pth'],
+        'config_file': 'models/cvpr2023.yaml',
+        'param': False,
+    },
+    'PersNet_Paramnet-GSV-uncentered': {
+        'weights': ['https://www.dropbox.com/s/ufdadxigewakzlz/paramnet_gsv_rpfpp.pth'],
+        'opts': ['MODEL.WEIGHTS', 'models/paramnet_gsv_rpfpp.pth'],
+        'config_file': 'models/paramnet_gsv_rpfpp.yaml',
+        'param': True,
+    },
+    # trained on GSV dataset, predicts Perspective Fields + camera parameters (roll, pitch, fov), assuming centered principal point
+    'PersNet_Paramnet-GSV-centered': {
+        'weights': ['https://www.dropbox.com/s/g6xwbgnkggapyeu/paramnet_gsv_rpf.pth'],
+        'opts': ['MODEL.WEIGHTS', 'models/paramnet_gsv_rpf.pth'],
+        'config_file': 'models/paramnet_gsv_rpf.yaml',
+        'param': True,
+    },
+}
+for model_id in model_zoo[model_id]:
+    html = model_zoo[model_id]['weights']
+    if not os.path.exists(os.path.join('models', html.split('/')[-1])):
+        os.system(f"wget -P models/ {html}")
+
+info = """Select model\n"""
+gr.Interface(
+    fn=inference,
+    inputs=[
+        "image", 
+        gr.Radio(
+            list(model_zoo.keys()), 
+            value=list(sorted(model_zoo.keys()))[0], 
+            label="Model", 
+            info=info,
+        ),
+    ],
+    outputs=[gr.Image(label='Perspective Fields'), gr.Textbox(label='Pred Camera Parameters')],
+    title=title,
+    description=description,
+    article=article,
+    examples=examples,
+).launch(share=True)

models/cvpr2023.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7107c87663d08cd45159f5c36913bba4c5e42c6d424d065c7e662481e88bab4
+size 307275893

models/cvpr2023.yaml

@@ -0,0 +1,391 @@
+CUDNN_BENCHMARK: false
+DATALOADER:
+  ASPECT_RATIO_GROUPING: true
+  AUGMENTATION: true
+  AUGMENTATION_FUN: default
+  AUGMENTATION_TYPE: geometry
+  FILTER_EMPTY_ANNOTATIONS: true
+  NUM_WORKERS: 8
+  REPEAT_THRESHOLD: 0.0
+  RESIZE:
+  - 320
+  - 320
+  SAMPLER_TRAIN: TrainingSampler
+DATASETS:
+  PRECOMPUTED_PROPOSAL_TOPK_TEST: 1000
+  PRECOMPUTED_PROPOSAL_TOPK_TRAIN: 2000
+  PROPOSAL_FILES_TEST: []
+  PROPOSAL_FILES_TRAIN: []
+  TEST:
+  - stanford2d3d_test
+  - tartanair_test
+  TRAIN:
+  - cities360_train
+  - rgbdpano_train
+DEBUG_ON: false
+GLOBAL:
+  HACK: 1.0
+INPUT:
+  CROP:
+    ENABLED: false
+    SIZE:
+    - 0.9
+    - 0.9
+    TYPE: relative_range
+  FORMAT: BGR
+  MASK_FORMAT: polygon
+  MAX_SIZE_TEST: 1333
+  MAX_SIZE_TRAIN: 1333
+  MIN_SIZE_TEST: 800
+  MIN_SIZE_TRAIN:
+  - 800
+  MIN_SIZE_TRAIN_SAMPLING: choice
+  ONLINE_CROP: false
+  RANDOM_FLIP: horizontal
+MODEL:
+  ANCHOR_GENERATOR:
+    ANGLES:
+    - - -90
+      - 0
+      - 90
+    ASPECT_RATIOS:
+    - - 0.5
+      - 1.0
+      - 2.0
+    NAME: DefaultAnchorGenerator
+    OFFSET: 0.0
+    SIZES:
+    - - 32
+      - 64
+      - 128
+      - 256
+      - 512
+  BACKBONE:
+    FREEZE_AT: 2
+    NAME: build_mit_backbone
+  CENTER_ON: false
+  DEVICE: cuda
+  FPN:
+    FUSE_TYPE: sum
+    IN_FEATURES: []
+    NORM: ''
+    OUT_CHANNELS: 256
+  FPN_CENTER_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: 360
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: CenterFPNHead
+    NORM: GN
+    NUM_CLASSES: 30
+  FPN_GRAVITY_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: 360
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: GravityFPNHead
+    NORM: GN
+    NUM_CLASSES: 361
+  FPN_HEADS:
+    NAME: StandardFPNHeads
+  FPN_LATITUDE_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: -1
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: LatitudeFPNHead
+    NORM: GN
+    NUM_CLASSES: 9
+  FREEZE: []
+  GRAVITY_DECODER:
+    IGNORE_VALUE: 72
+    LOSS_TYPE: classification
+    LOSS_WEIGHT: 1.0
+    NAME: GravityDecoder
+    NUM_CLASSES: 73
+  GRAVITY_ON: true
+  HEIGHT_DECODER:
+    LOSS_WEIGHT: 1.0
+    NAME: HeightDecoder
+  HEIGHT_ON: false
+  KEYPOINT_ON: false
+  LATITUDE_DECODER:
+    IGNORE_VALUE: -1
+    LOSS_TYPE: classification
+    LOSS_WEIGHT: 1.0
+    NAME: LatitudeDecoder
+    NUM_CLASSES: 180
+  LATITUDE_ON: true
+  LOAD_PROPOSALS: false
+  MASK_ON: false
+  META_ARCHITECTURE: PersFormer
+  PANOPTIC_FPN:
+    COMBINE:
+      ENABLED: true
+      INSTANCES_CONFIDENCE_THRESH: 0.5
+      OVERLAP_THRESH: 0.5
+      STUFF_AREA_LIMIT: 4096
+    INSTANCE_LOSS_WEIGHT: 1.0
+  PARAM_DECODER:
+    DEBUG_LAT: false
+    DEBUG_UP: false
+    INPUT_SIZE: 320
+    LOSS_TYPE: regression
+    LOSS_WEIGHT: 1.0
+    NAME: ParamNet
+    PREDICT_PARAMS:
+    - roll
+    - pitch
+    - rel_focal
+    - rel_cx
+    - rel_cy
+    SYNTHETIC_PRETRAIN: false
+  PERSFORMER_HEADS:
+    NAME: StandardPersformerHeads
+  PIXEL_MEAN:
+  - 103.53
+  - 116.28
+  - 123.675
+  PIXEL_STD:
+  - 1.0
+  - 1.0
+  - 1.0
+  PROPOSAL_GENERATOR:
+    MIN_SIZE: 0
+    NAME: RPN
+  RECOVER_PP: false
+  RECOVER_RPF: false
+  RESNETS:
+    DEFORM_MODULATED: false
+    DEFORM_NUM_GROUPS: 1
+    DEFORM_ON_PER_STAGE:
+    - false
+    - false
+    - false
+    - false
+    DEPTH: 50
+    NORM: FrozenBN
+    NUM_GROUPS: 1
+    OUT_FEATURES:
+    - res4
+    RES2_OUT_CHANNELS: 256
+    RES5_DILATION: 1
+    STEM_OUT_CHANNELS: 64
+    STRIDE_IN_1X1: true
+    WIDTH_PER_GROUP: 64
+  RETINANET:
+    BBOX_REG_LOSS_TYPE: smooth_l1
+    BBOX_REG_WEIGHTS: &id002
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    FOCAL_LOSS_ALPHA: 0.25
+    FOCAL_LOSS_GAMMA: 2.0
+    IN_FEATURES:
+    - p3
+    - p4
+    - p5
+    - p6
+    - p7
+    IOU_LABELS:
+    - 0
+    - -1
+    - 1
+    IOU_THRESHOLDS:
+    - 0.4
+    - 0.5
+    NMS_THRESH_TEST: 0.5
+    NORM: ''
+    NUM_CLASSES: 80
+    NUM_CONVS: 4
+    PRIOR_PROB: 0.01
+    SCORE_THRESH_TEST: 0.05
+    SMOOTH_L1_LOSS_BETA: 0.1
+    TOPK_CANDIDATES_TEST: 1000
+  ROI_BOX_CASCADE_HEAD:
+    BBOX_REG_WEIGHTS:
+    - &id001
+      - 10.0
+      - 10.0
+      - 5.0
+      - 5.0
+    - - 20.0
+      - 20.0
+      - 10.0
+      - 10.0
+    - - 30.0
+      - 30.0
+      - 15.0
+      - 15.0
+    IOUS:
+    - 0.5
+    - 0.6
+    - 0.7
+  ROI_BOX_HEAD:
+    BBOX_REG_LOSS_TYPE: smooth_l1
+    BBOX_REG_LOSS_WEIGHT: 1.0
+    BBOX_REG_WEIGHTS: *id001
+    CLS_AGNOSTIC_BBOX_REG: false
+    CONV_DIM: 256
+    FC_DIM: 1024
+    NAME: ''
+    NORM: ''
+    NUM_CONV: 0
+    NUM_FC: 0
+    POOLER_RESOLUTION: 14
+    POOLER_SAMPLING_RATIO: 0
+    POOLER_TYPE: ROIAlignV2
+    SMOOTH_L1_BETA: 0.0
+    TRAIN_ON_PRED_BOXES: false
+  ROI_HEADS:
+    BATCH_SIZE_PER_IMAGE: 512
+    IN_FEATURES:
+    - res4
+    IOU_LABELS:
+    - 0
+    - 1
+    IOU_THRESHOLDS:
+    - 0.5
+    NAME: Res5ROIHeads
+    NMS_THRESH_TEST: 0.5
+    NUM_CLASSES: 80
+    POSITIVE_FRACTION: 0.25
+    PROPOSAL_APPEND_GT: true
+    SCORE_THRESH_TEST: 0.05
+  ROI_KEYPOINT_HEAD:
+    CONV_DIMS:
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    LOSS_WEIGHT: 1.0
+    MIN_KEYPOINTS_PER_IMAGE: 1
+    NAME: KRCNNConvDeconvUpsampleHead
+    NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS: true
+    NUM_KEYPOINTS: 17
+    POOLER_RESOLUTION: 14
+    POOLER_SAMPLING_RATIO: 0
+    POOLER_TYPE: ROIAlignV2
+  ROI_MASK_HEAD:
+    CLS_AGNOSTIC_MASK: false
+    CONV_DIM: 256
+    NAME: MaskRCNNConvUpsampleHead
+    NORM: ''
+    NUM_CONV: 0
+    POOLER_RESOLUTION: 14
+    POOLER_SAMPLING_RATIO: 0
+    POOLER_TYPE: ROIAlignV2
+  RPN:
+    BATCH_SIZE_PER_IMAGE: 256
+    BBOX_REG_LOSS_TYPE: smooth_l1
+    BBOX_REG_LOSS_WEIGHT: 1.0
+    BBOX_REG_WEIGHTS: *id002
+    BOUNDARY_THRESH: -1
+    CONV_DIMS:
+    - -1
+    HEAD_NAME: StandardRPNHead
+    IN_FEATURES:
+    - res4
+    IOU_LABELS:
+    - 0
+    - -1
+    - 1
+    IOU_THRESHOLDS:
+    - 0.3
+    - 0.7
+    LOSS_WEIGHT: 1.0
+    NMS_THRESH: 0.7
+    POSITIVE_FRACTION: 0.5
+    POST_NMS_TOPK_TEST: 1000
+    POST_NMS_TOPK_TRAIN: 2000
+    PRE_NMS_TOPK_TEST: 6000
+    PRE_NMS_TOPK_TRAIN: 12000
+    SMOOTH_L1_BETA: 0.0
+  SEM_SEG_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: 255
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: SemSegFPNHead
+    NORM: GN
+    NUM_CLASSES: 54
+  WEIGHTS: init_model_weights/ade_pretrained.pth
+OUTPUT_DIR: /home/jinlinyi/exp/densefield/e08_persformer/e15_latiup_mixed_v3
+OVERFIT_ON: false
+SEED: -1
+SOLVER:
+  AMP:
+    ENABLED: false
+  BASE_LR: 0.001
+  BIAS_LR_FACTOR: 1.0
+  CHECKPOINT_PERIOD: 2000
+  CLIP_GRADIENTS:
+    CLIP_TYPE: value
+    CLIP_VALUE: 1.0
+    ENABLED: false
+    NORM_TYPE: 2.0
+  GAMMA: 0.1
+  IMS_PER_BATCH: 64
+  LR_SCHEDULER_NAME: WarmupMultiStepLR
+  MAX_ITER: 90000
+  MOMENTUM: 0.9
+  NESTEROV: false
+  REFERENCE_WORLD_SIZE: 0
+  STEPS:
+  - 40000
+  - 60000
+  WARMUP_FACTOR: 0.001
+  WARMUP_ITERS: 1000
+  WARMUP_METHOD: linear
+  WEIGHT_DECAY: 0.0001
+  WEIGHT_DECAY_BIAS: null
+  WEIGHT_DECAY_NORM: 0.0
+TEST:
+  AUG:
+    ENABLED: false
+    FLIP: true
+    MAX_SIZE: 4000
+    MIN_SIZES:
+    - 400
+    - 500
+    - 600
+    - 700
+    - 800
+    - 900
+    - 1000
+    - 1100
+    - 1200
+  DETECTIONS_PER_IMAGE: 100
+  EVAL_PERIOD: 2000
+  EXPECTED_RESULTS: []
+  KEYPOINT_OKS_SIGMAS: []
+  PRECISE_BN:
+    ENABLED: false
+    NUM_ITER: 200
+VERSION: 2
+VIS_PERIOD: 200

models/paramnet_gsv_rpf.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3965db8523f573f8037b43696cd08d1e63a498218e28a4ee7ebd50b373bbd56d
+size 418607881

models/paramnet_gsv_rpf.yaml

@@ -0,0 +1,377 @@
+CUDNN_BENCHMARK: false
+DATALOADER:
+  ASPECT_RATIO_GROUPING: true
+  AUGMENTATION: true
+  AUGMENTATION_TYPE: geometry
+  FILTER_EMPTY_ANNOTATIONS: true
+  NUM_WORKERS: 8
+  REPEAT_THRESHOLD: 0.0
+  RESIZE:
+  - 320
+  - 320
+  SAMPLER_TRAIN: TrainingSampler
+DATASETS:
+  PRECOMPUTED_PROPOSAL_TOPK_TEST: 1000
+  PRECOMPUTED_PROPOSAL_TOPK_TRAIN: 2000
+  PROPOSAL_FILES_TEST: []
+  PROPOSAL_FILES_TRAIN: []
+  TEST:
+  - gsv_val
+  - gsv_test
+  TRAIN:
+  - gsv_train
+GLOBAL:
+  HACK: 1.0
+INPUT:
+  CROP:
+    ENABLED: false
+    SIZE:
+    - 0.9
+    - 0.9
+    TYPE: relative_range
+  FORMAT: BGR
+  MASK_FORMAT: polygon
+  MAX_SIZE_TEST: 1333
+  MAX_SIZE_TRAIN: 1333
+  MIN_SIZE_TEST: 800
+  MIN_SIZE_TRAIN:
+  - 800
+  MIN_SIZE_TRAIN_SAMPLING: choice
+  ONLINE_CROP: false
+  RANDOM_FLIP: horizontal
+MODEL:
+  ANCHOR_GENERATOR:
+    ANGLES:
+    - - -90
+      - 0
+      - 90
+    ASPECT_RATIOS:
+    - - 0.5
+      - 1.0
+      - 2.0
+    NAME: DefaultAnchorGenerator
+    OFFSET: 0.0
+    SIZES:
+    - - 32
+      - 64
+      - 128
+      - 256
+      - 512
+  BACKBONE:
+    FREEZE_AT: 2
+    NAME: build_mit_backbone
+  CENTER_ON: false
+  DEVICE: cuda
+  FPN:
+    FUSE_TYPE: sum
+    IN_FEATURES: []
+    NORM: ''
+    OUT_CHANNELS: 256
+  FPN_CENTER_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: 360
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: CenterFPNHead
+    NORM: GN
+    NUM_CLASSES: 30
+  FPN_GRAVITY_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: 360
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: GravityFPNHead
+    NORM: GN
+    NUM_CLASSES: 361
+  FPN_HEADS:
+    NAME: StandardFPNHeads
+  FPN_LATITUDE_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: -1
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: LatitudeFPNHead
+    NORM: GN
+    NUM_CLASSES: 9
+  FREEZE: []
+  GRAVITY_DECODER:
+    IGNORE_VALUE: 72
+    LOSS_TYPE: regression
+    LOSS_WEIGHT: 1.0
+    NAME: GravityDecoder
+    NUM_CLASSES: 73
+  GRAVITY_ON: true
+  HEIGHT_DECODER:
+    LOSS_WEIGHT: 1.0
+    NAME: HeightDecoder
+  HEIGHT_ON: false
+  KEYPOINT_ON: false
+  LATITUDE_DECODER:
+    IGNORE_VALUE: -1
+    LOSS_TYPE: regression
+    LOSS_WEIGHT: 1.0
+    NAME: LatitudeDecoder
+    NUM_CLASSES: 1
+  LATITUDE_ON: true
+  LOAD_PROPOSALS: false
+  MASK_ON: false
+  META_ARCHITECTURE: PersFormer
+  PANOPTIC_FPN:
+    COMBINE:
+      ENABLED: true
+      INSTANCES_CONFIDENCE_THRESH: 0.5
+      OVERLAP_THRESH: 0.5
+      STUFF_AREA_LIMIT: 4096
+    INSTANCE_LOSS_WEIGHT: 1.0
+  PARAM_DECODER:
+    LOSS_TYPE: regression
+    LOSS_WEIGHT: 1.0
+    NAME: ParamNet
+  PERSFORMER_HEADS:
+    NAME: StandardPersformerHeads
+  PIXEL_MEAN:
+  - 103.53
+  - 116.28
+  - 123.675
+  PIXEL_STD:
+  - 1.0
+  - 1.0
+  - 1.0
+  PROPOSAL_GENERATOR:
+    MIN_SIZE: 0
+    NAME: RPN
+  RECOVER_PP: false
+  RECOVER_RPF: true
+  RESNETS:
+    DEFORM_MODULATED: false
+    DEFORM_NUM_GROUPS: 1
+    DEFORM_ON_PER_STAGE:
+    - false
+    - false
+    - false
+    - false
+    DEPTH: 50
+    NORM: FrozenBN
+    NUM_GROUPS: 1
+    OUT_FEATURES:
+    - res4
+    RES2_OUT_CHANNELS: 256
+    RES5_DILATION: 1
+    STEM_OUT_CHANNELS: 64
+    STRIDE_IN_1X1: true
+    WIDTH_PER_GROUP: 64
+  RETINANET:
+    BBOX_REG_LOSS_TYPE: smooth_l1
+    BBOX_REG_WEIGHTS: &id002
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    FOCAL_LOSS_ALPHA: 0.25
+    FOCAL_LOSS_GAMMA: 2.0
+    IN_FEATURES:
+    - p3
+    - p4
+    - p5
+    - p6
+    - p7
+    IOU_LABELS:
+    - 0
+    - -1
+    - 1
+    IOU_THRESHOLDS:
+    - 0.4
+    - 0.5
+    NMS_THRESH_TEST: 0.5
+    NORM: ''
+    NUM_CLASSES: 80
+    NUM_CONVS: 4
+    PRIOR_PROB: 0.01
+    SCORE_THRESH_TEST: 0.05
+    SMOOTH_L1_LOSS_BETA: 0.1
+    TOPK_CANDIDATES_TEST: 1000
+  ROI_BOX_CASCADE_HEAD:
+    BBOX_REG_WEIGHTS:
+    - &id001
+      - 10.0
+      - 10.0
+      - 5.0
+      - 5.0
+    - - 20.0
+      - 20.0
+      - 10.0
+      - 10.0
+    - - 30.0
+      - 30.0
+      - 15.0
+      - 15.0
+    IOUS:
+    - 0.5
+    - 0.6
+    - 0.7
+  ROI_BOX_HEAD:
+    BBOX_REG_LOSS_TYPE: smooth_l1
+    BBOX_REG_LOSS_WEIGHT: 1.0
+    BBOX_REG_WEIGHTS: *id001
+    CLS_AGNOSTIC_BBOX_REG: false
+    CONV_DIM: 256
+    FC_DIM: 1024
+    NAME: ''
+    NORM: ''
+    NUM_CONV: 0
+    NUM_FC: 0
+    POOLER_RESOLUTION: 14
+    POOLER_SAMPLING_RATIO: 0
+    POOLER_TYPE: ROIAlignV2
+    SMOOTH_L1_BETA: 0.0
+    TRAIN_ON_PRED_BOXES: false
+  ROI_HEADS:
+    BATCH_SIZE_PER_IMAGE: 512
+    IN_FEATURES:
+    - res4
+    IOU_LABELS:
+    - 0
+    - 1
+    IOU_THRESHOLDS:
+    - 0.5
+    NAME: Res5ROIHeads
+    NMS_THRESH_TEST: 0.5
+    NUM_CLASSES: 80
+    POSITIVE_FRACTION: 0.25
+    PROPOSAL_APPEND_GT: true
+    SCORE_THRESH_TEST: 0.05
+  ROI_KEYPOINT_HEAD:
+    CONV_DIMS:
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    LOSS_WEIGHT: 1.0
+    MIN_KEYPOINTS_PER_IMAGE: 1
+    NAME: KRCNNConvDeconvUpsampleHead
+    NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS: true
+    NUM_KEYPOINTS: 17
+    POOLER_RESOLUTION: 14
+    POOLER_SAMPLING_RATIO: 0
+    POOLER_TYPE: ROIAlignV2
+  ROI_MASK_HEAD:
+    CLS_AGNOSTIC_MASK: false
+    CONV_DIM: 256
+    NAME: MaskRCNNConvUpsampleHead
+    NORM: ''
+    NUM_CONV: 0
+    POOLER_RESOLUTION: 14
+    POOLER_SAMPLING_RATIO: 0
+    POOLER_TYPE: ROIAlignV2
+  RPN:
+    BATCH_SIZE_PER_IMAGE: 256
+    BBOX_REG_LOSS_TYPE: smooth_l1
+    BBOX_REG_LOSS_WEIGHT: 1.0
+    BBOX_REG_WEIGHTS: *id002
+    BOUNDARY_THRESH: -1
+    CONV_DIMS:
+    - -1
+    HEAD_NAME: StandardRPNHead
+    IN_FEATURES:
+    - res4
+    IOU_LABELS:
+    - 0
+    - -1
+    - 1
+    IOU_THRESHOLDS:
+    - 0.3
+    - 0.7
+    LOSS_WEIGHT: 1.0
+    NMS_THRESH: 0.7
+    POSITIVE_FRACTION: 0.5
+    POST_NMS_TOPK_TEST: 1000
+    POST_NMS_TOPK_TRAIN: 2000
+    PRE_NMS_TOPK_TEST: 6000
+    PRE_NMS_TOPK_TRAIN: 12000
+    SMOOTH_L1_BETA: 0.0
+  SEM_SEG_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: 255
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: SemSegFPNHead
+    NORM: GN
+    NUM_CLASSES: 54
+  WEIGHTS: /home/jinlinyi/exp/densefield/e32_gsv_regress_ade/e04-lr-3-rebalanced/model_final.pth
+OUTPUT_DIR: /home/jinlinyi/exp/densefield/e33_recover_rpf/all-adepretrained-lr-2
+SEED: -1
+SOLVER:
+  AMP:
+    ENABLED: false
+  BASE_LR: 0.01
+  BIAS_LR_FACTOR: 1.0
+  CHECKPOINT_PERIOD: 500
+  CLIP_GRADIENTS:
+    CLIP_TYPE: value
+    CLIP_VALUE: 1.0
+    ENABLED: false
+    NORM_TYPE: 2.0
+  GAMMA: 0.1
+  IMS_PER_BATCH: 64
+  LR_SCHEDULER_NAME: WarmupMultiStepLR
+  MAX_ITER: 90000
+  MOMENTUM: 0.9
+  NESTEROV: false
+  REFERENCE_WORLD_SIZE: 0
+  STEPS:
+  - 40000
+  - 60000
+  WARMUP_FACTOR: 0.001
+  WARMUP_ITERS: 1000
+  WARMUP_METHOD: linear
+  WEIGHT_DECAY: 0.0001
+  WEIGHT_DECAY_BIAS: null
+  WEIGHT_DECAY_NORM: 0.0
+TEST:
+  AUG:
+    ENABLED: false
+    FLIP: true
+    MAX_SIZE: 4000
+    MIN_SIZES:
+    - 400
+    - 500
+    - 600
+    - 700
+    - 800
+    - 900
+    - 1000
+    - 1100
+    - 1200
+  DETECTIONS_PER_IMAGE: 100
+  EVAL_PERIOD: 500
+  EXPECTED_RESULTS: []
+  KEYPOINT_OKS_SIGMAS: []
+  PRECISE_BN:
+    ENABLED: false
+    NUM_ITER: 200
+VERSION: 2
+VIS_PERIOD: 500

models/paramnet_gsv_rpfpp.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f00aae81c9e0247b96ae9d6dad13dc2395b3078f3b4229dc19cfdd6a13b94ce
+size 418607881

models/paramnet_gsv_rpfpp.yaml

@@ -0,0 +1,389 @@
+CUDNN_BENCHMARK: false
+DATALOADER:
+  ASPECT_RATIO_GROUPING: true
+  AUGMENTATION: true
+  AUGMENTATION_FUN: uniform_vfov_crop_resize
+  AUGMENTATION_TYPE: geometry
+  FILTER_EMPTY_ANNOTATIONS: true
+  NUM_WORKERS: 8
+  REPEAT_THRESHOLD: 0.0
+  RESIZE:
+  - 320
+  - 320
+  SAMPLER_TRAIN: TrainingSampler
+DATASETS:
+  PRECOMPUTED_PROPOSAL_TOPK_TEST: 1000
+  PRECOMPUTED_PROPOSAL_TOPK_TRAIN: 2000
+  PROPOSAL_FILES_TEST: []
+  PROPOSAL_FILES_TRAIN: []
+  TEST:
+  - gsv_test_crop_uniform
+  TRAIN:
+  - gsv_train
+DEBUG_ON: false
+GLOBAL:
+  HACK: 1.0
+INPUT:
+  CROP:
+    ENABLED: false
+    SIZE:
+    - 0.9
+    - 0.9
+    TYPE: relative_range
+  FORMAT: BGR
+  MASK_FORMAT: polygon
+  MAX_SIZE_TEST: 1333
+  MAX_SIZE_TRAIN: 1333
+  MIN_SIZE_TEST: 800
+  MIN_SIZE_TRAIN:
+  - 800
+  MIN_SIZE_TRAIN_SAMPLING: choice
+  ONLINE_CROP: false
+  RANDOM_FLIP: horizontal
+MODEL:
+  ANCHOR_GENERATOR:
+    ANGLES:
+    - - -90
+      - 0
+      - 90
+    ASPECT_RATIOS:
+    - - 0.5
+      - 1.0
+      - 2.0
+    NAME: DefaultAnchorGenerator
+    OFFSET: 0.0
+    SIZES:
+    - - 32
+      - 64
+      - 128
+      - 256
+      - 512
+  BACKBONE:
+    FREEZE_AT: 2
+    NAME: build_mit_backbone
+  CENTER_ON: false
+  DEVICE: cuda
+  FPN:
+    FUSE_TYPE: sum
+    IN_FEATURES: []
+    NORM: ''
+    OUT_CHANNELS: 256
+  FPN_CENTER_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: 360
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: CenterFPNHead
+    NORM: GN
+    NUM_CLASSES: 30
+  FPN_GRAVITY_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: 360
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: GravityFPNHead
+    NORM: GN
+    NUM_CLASSES: 361
+  FPN_HEADS:
+    NAME: StandardFPNHeads
+  FPN_LATITUDE_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: -1
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: LatitudeFPNHead
+    NORM: GN
+    NUM_CLASSES: 9
+  FREEZE: []
+  GRAVITY_DECODER:
+    IGNORE_VALUE: 72
+    LOSS_TYPE: regression
+    LOSS_WEIGHT: 1.0
+    NAME: GravityDecoder
+    NUM_CLASSES: 73
+  GRAVITY_ON: true
+  HEIGHT_DECODER:
+    LOSS_WEIGHT: 1.0
+    NAME: HeightDecoder
+  HEIGHT_ON: false
+  KEYPOINT_ON: false
+  LATITUDE_DECODER:
+    IGNORE_VALUE: -1
+    LOSS_TYPE: regression
+    LOSS_WEIGHT: 1.0
+    NAME: LatitudeDecoder
+    NUM_CLASSES: 1
+  LATITUDE_ON: true
+  LOAD_PROPOSALS: false
+  MASK_ON: false
+  META_ARCHITECTURE: PersFormer
+  PANOPTIC_FPN:
+    COMBINE:
+      ENABLED: true
+      INSTANCES_CONFIDENCE_THRESH: 0.5
+      OVERLAP_THRESH: 0.5
+      STUFF_AREA_LIMIT: 4096
+    INSTANCE_LOSS_WEIGHT: 1.0
+  PARAM_DECODER:
+    DEBUG_LAT: false
+    DEBUG_UP: false
+    INPUT_SIZE: 64
+    LOSS_TYPE: regression
+    LOSS_WEIGHT: 0.1
+    NAME: ParamNetConvNextRegress
+    PREDICT_PARAMS:
+    - roll
+    - pitch
+    - general_vfov
+    - rel_cx
+    - rel_cy
+    SYNTHETIC_PRETRAIN: false
+  PERSFORMER_HEADS:
+    NAME: StandardPersformerHeads
+  PIXEL_MEAN:
+  - 103.53
+  - 116.28
+  - 123.675
+  PIXEL_STD:
+  - 1.0
+  - 1.0
+  - 1.0
+  PROPOSAL_GENERATOR:
+    MIN_SIZE: 0
+    NAME: RPN
+  RECOVER_PP: true
+  RECOVER_RPF: true
+  RESNETS:
+    DEFORM_MODULATED: false
+    DEFORM_NUM_GROUPS: 1
+    DEFORM_ON_PER_STAGE:
+    - false
+    - false
+    - false
+    - false
+    DEPTH: 50
+    NORM: FrozenBN
+    NUM_GROUPS: 1
+    OUT_FEATURES:
+    - res4
+    RES2_OUT_CHANNELS: 256
+    RES5_DILATION: 1
+    STEM_OUT_CHANNELS: 64
+    STRIDE_IN_1X1: true
+    WIDTH_PER_GROUP: 64
+  RETINANET:
+    BBOX_REG_LOSS_TYPE: smooth_l1
+    BBOX_REG_WEIGHTS: &id002
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    FOCAL_LOSS_ALPHA: 0.25
+    FOCAL_LOSS_GAMMA: 2.0
+    IN_FEATURES:
+    - p3
+    - p4
+    - p5
+    - p6
+    - p7
+    IOU_LABELS:
+    - 0
+    - -1
+    - 1
+    IOU_THRESHOLDS:
+    - 0.4
+    - 0.5
+    NMS_THRESH_TEST: 0.5
+    NORM: ''
+    NUM_CLASSES: 80
+    NUM_CONVS: 4
+    PRIOR_PROB: 0.01
+    SCORE_THRESH_TEST: 0.05
+    SMOOTH_L1_LOSS_BETA: 0.1
+    TOPK_CANDIDATES_TEST: 1000
+  ROI_BOX_CASCADE_HEAD:
+    BBOX_REG_WEIGHTS:
+    - &id001
+      - 10.0
+      - 10.0
+      - 5.0
+      - 5.0
+    - - 20.0
+      - 20.0
+      - 10.0
+      - 10.0
+    - - 30.0
+      - 30.0
+      - 15.0
+      - 15.0
+    IOUS:
+    - 0.5
+    - 0.6
+    - 0.7
+  ROI_BOX_HEAD:
+    BBOX_REG_LOSS_TYPE: smooth_l1
+    BBOX_REG_LOSS_WEIGHT: 1.0
+    BBOX_REG_WEIGHTS: *id001
+    CLS_AGNOSTIC_BBOX_REG: false
+    CONV_DIM: 256
+    FC_DIM: 1024
+    NAME: ''
+    NORM: ''
+    NUM_CONV: 0
+    NUM_FC: 0
+    POOLER_RESOLUTION: 14
+    POOLER_SAMPLING_RATIO: 0
+    POOLER_TYPE: ROIAlignV2
+    SMOOTH_L1_BETA: 0.0
+    TRAIN_ON_PRED_BOXES: false
+  ROI_HEADS:
+    BATCH_SIZE_PER_IMAGE: 512
+    IN_FEATURES:
+    - res4
+    IOU_LABELS:
+    - 0
+    - 1
+    IOU_THRESHOLDS:
+    - 0.5
+    NAME: Res5ROIHeads
+    NMS_THRESH_TEST: 0.5
+    NUM_CLASSES: 80
+    POSITIVE_FRACTION: 0.25
+    PROPOSAL_APPEND_GT: true
+    SCORE_THRESH_TEST: 0.05
+  ROI_KEYPOINT_HEAD:
+    CONV_DIMS:
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    - 512
+    LOSS_WEIGHT: 1.0
+    MIN_KEYPOINTS_PER_IMAGE: 1
+    NAME: KRCNNConvDeconvUpsampleHead
+    NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS: true
+    NUM_KEYPOINTS: 17
+    POOLER_RESOLUTION: 14
+    POOLER_SAMPLING_RATIO: 0
+    POOLER_TYPE: ROIAlignV2
+  ROI_MASK_HEAD:
+    CLS_AGNOSTIC_MASK: false
+    CONV_DIM: 256
+    NAME: MaskRCNNConvUpsampleHead
+    NORM: ''
+    NUM_CONV: 0
+    POOLER_RESOLUTION: 14
+    POOLER_SAMPLING_RATIO: 0
+    POOLER_TYPE: ROIAlignV2
+  RPN:
+    BATCH_SIZE_PER_IMAGE: 256
+    BBOX_REG_LOSS_TYPE: smooth_l1
+    BBOX_REG_LOSS_WEIGHT: 1.0
+    BBOX_REG_WEIGHTS: *id002
+    BOUNDARY_THRESH: -1
+    CONV_DIMS:
+    - -1
+    HEAD_NAME: StandardRPNHead
+    IN_FEATURES:
+    - res4
+    IOU_LABELS:
+    - 0
+    - -1
+    - 1
+    IOU_THRESHOLDS:
+    - 0.3
+    - 0.7
+    LOSS_WEIGHT: 1.0
+    NMS_THRESH: 0.7
+    POSITIVE_FRACTION: 0.5
+    POST_NMS_TOPK_TEST: 1000
+    POST_NMS_TOPK_TRAIN: 2000
+    PRE_NMS_TOPK_TEST: 6000
+    PRE_NMS_TOPK_TRAIN: 12000
+    SMOOTH_L1_BETA: 0.0
+  SEM_SEG_HEAD:
+    COMMON_STRIDE: 4
+    CONVS_DIM: 128
+    IGNORE_VALUE: 255
+    IN_FEATURES:
+    - p2
+    - p3
+    - p4
+    - p5
+    LOSS_WEIGHT: 1.0
+    NAME: SemSegFPNHead
+    NORM: GN
+    NUM_CLASSES: 54
+  WEIGHTS: /home/jinlinyi/exp/densefield/e32_gsv_regress_ade/e04-lr-3-rebalanced/model_final.pth
+OUTPUT_DIR: /home/jinlinyi/exp/densefield/e36_recover_rpfpp_v3_small/regression-all-lr2-rebalanced
+OVERFIT_ON: false
+SEED: -1
+SOLVER:
+  AMP:
+    ENABLED: false
+  BASE_LR: 0.01
+  BIAS_LR_FACTOR: 1.0
+  CHECKPOINT_PERIOD: 500
+  CLIP_GRADIENTS:
+    CLIP_TYPE: value
+    CLIP_VALUE: 1.0
+    ENABLED: false
+    NORM_TYPE: 2.0
+  GAMMA: 0.1
+  IMS_PER_BATCH: 64
+  LR_SCHEDULER_NAME: WarmupCosineLR
+  MAX_ITER: 90000
+  MOMENTUM: 0.9
+  NESTEROV: false
+  REFERENCE_WORLD_SIZE: 0
+  STEPS:
+  - 40000
+  - 60000
+  WARMUP_FACTOR: 0.001
+  WARMUP_ITERS: 1000
+  WARMUP_METHOD: linear
+  WEIGHT_DECAY: 0.0001
+  WEIGHT_DECAY_BIAS: null
+  WEIGHT_DECAY_NORM: 0.0
+TEST:
+  AUG:
+    ENABLED: false
+    FLIP: true
+    MAX_SIZE: 4000
+    MIN_SIZES:
+    - 400
+    - 500
+    - 600
+    - 700
+    - 800
+    - 900
+    - 1000
+    - 1100
+    - 1200
+  DETECTIONS_PER_IMAGE: 100
+  EVAL_PERIOD: 500
+  EXPECTED_RESULTS: []
+  KEYPOINT_OKS_SIGMAS: []
+  PRECISE_BN:
+    ENABLED: false
+    NUM_ITER: 200
+VERSION: 2
+VIS_PERIOD: 500

requirements.txt

@@ -0,0 +1,3 @@
+--extra-index-url https://download.pytorch.org/whl/cu113
+torch==1.10.0+cu113
+torchvision==0.11.1+cu113