-
Notifications
You must be signed in to change notification settings - Fork 1
/
vqa_challenge_2019_leaderboard_updated.json
1 lines (1 loc) · 18.2 KB
/
vqa_challenge_2019_leaderboard_updated.json
1
[{"team_members": "", "challenge": {"overall": 71.51, "perAnswerType": {"other": 62.62, "number": 53.7, "yes/no": 86.45}}, "dev": {"overall": 71.21, "perAnswerType": {"other": 62.37, "number": 53.74, "yes/no": 86.22}}, "standard": {"overall": 71.52, "perAnswerType": {"other": 62.51, "number": 53.55, "yes/no": 86.61}}, "team_name_order": 1, "team_name": "508SB", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 70.98, "perAnswerType": {"other": 62.12, "number": 52.89, "yes/no": 85.97}}, "dev": {"overall": 70.68, "perAnswerType": {"other": 61.86, "number": 52.66, "yes/no": 85.81}}, "standard": {"overall": 71.03, "perAnswerType": {"other": 62.08, "number": 52.36, "yes/no": 86.22}}, "team_name_order": 2, "team_name": "508lh", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 71.52, "perAnswerType": {"other": 62.62, "number": 53.7, "yes/no": 86.47}}, "dev": {"overall": 71.21, "perAnswerType": {"other": 62.35, "number": 53.83, "yes/no": 86.21}}, "standard": {"overall": 71.53, "perAnswerType": {"other": 62.51, "number": 53.53, "yes/no": 86.64}}, "team_name_order": 3, "team_name": "508sc", "ref": "", "method": ""}, {"team_members": "Tuong Do (AIOZ), Huy Tran (AIOZ), Thanh-Toan Do (AIOZ), Erman Tjiputra (AIOZ), Quang D. Tran (AIOZ)", "challenge": {"overall": 73.04, "perAnswerType": {"other": 63.93, "number": 56.16, "yes/no": 87.99}}, "dev": {"overall": 72.75, "perAnswerType": {"other": 63.51, "number": 56.12, "yes/no": 87.96}}, "standard": {"overall": 72.93, "perAnswerType": {"other": 63.63, "number": 55.22, "yes/no": 88.26}}, "team_name_order": 4, "team_name": "AIOZ", "ref": "https://ai.aioz.io/posters/cvpr19/aioz_vqa_cvpr2019.pdf", "method": "We present \"Interaction learning with\nquestion-type awareness\" (i.e., IL-QTA model) for Visual Question Answering and the VQA Challenge 2019. From the starting point which is a re-implementation of the BAN model, we introduce an interaction learning that allows to leverage hypotheses from different attention mechanisms in an interacting manner. We also show that by making subtle but important changes to the component model architectures, leveraging the question-type information, and adding augmented test data with pseudo-labelling, we can significantly improve the performance of the current state-of-the-art BAN model on VQA v2.0 dataset from 70.35% to 72.08%."}, {"team_members": "Jin-Hwa Kim (SK T-Brain), Jaehyun Jun (SK T-Brain), Byoung-Tak Zhang (SNU)", "challenge": {"overall": 71.69, "perAnswerType": {"other": 62.36, "number": 54.92, "yes/no": 86.86}}, "dev": {"overall": 71.4, "perAnswerType": {"other": 62.08, "number": 54.94, "yes/no": 86.68}}, "standard": {"overall": 71.84, "perAnswerType": {"other": 62.45, "number": 54.37, "yes/no": 87.22}}, "team_name_order": 5, "team_name": "BAN", "ref": "http://papers.nips.cc/paper/7429-bilinear-attention-networks", "method": "An ensemble of fifteen eight-glimpse bilinear attention networks, which considers every interaction between question tokens and visual features, integrated with the counting module from Zhang et al. (2018). Image features are extracted by Bottom-up attention (Anderson et al., 2018). A part of Visual genome dataset (Krishna et al., 2017) is augmented. Runners-up in 2018 VQA Challenge."}, {"team_members": "", "challenge": {"overall": 69.32, "perAnswerType": {"other": 60.47, "number": 48.06, "yes/no": 85.16}}, "dev": {"overall": 68.98, "perAnswerType": {"other": 60.04, "number": 48.08, "yes/no": 85.05}}, "standard": {"overall": 69.28, "perAnswerType": {"other": 60.16, "number": 47.27, "yes/no": 85.55}}, "team_name_order": 6, "team_name": "CMU-LTI", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 68.49, "perAnswerType": {"other": 58.68, "number": 47.18, "yes/no": 85.44}}, "dev": {"overall": 68.31, "perAnswerType": {"other": 58.63, "number": 46.78, "yes/no": 85.39}}, "standard": {"overall": 68.45, "perAnswerType": {"other": 58.72, "number": 46.4, "yes/no": 85.42}}, "team_name_order": 7, "team_name": "Columbia", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 71.15, "perAnswerType": {"other": 61.63, "number": 54.76, "yes/no": 86.43}}, "dev": {"overall": 70.82, "perAnswerType": {"other": 61.3, "number": 54.85, "yes/no": 86.18}}, "standard": {"overall": 71.15, "perAnswerType": {"other": 61.54, "number": 54.13, "yes/no": 86.66}}, "team_name_order": 8, "team_name": "DeepBlueAI", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 65.25, "perAnswerType": {"other": 56.03, "number": 44.38, "yes/no": 81.41}}, "dev": {"overall": 64.89, "perAnswerType": {"other": 55.71, "number": 43.64, "yes/no": 81.32}}, "standard": {"overall": 65.27, "perAnswerType": {"other": 56.1, "number": 42.76, "yes/no": 81.73}}, "team_name_order": 9, "team_name": "Deepthinkers", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 71.36, "perAnswerType": {"other": 62.51, "number": 53.51, "yes/no": 86.29}}, "dev": {"overall": 71.12, "perAnswerType": {"other": 62.27, "number": 53.42, "yes/no": 86.18}}, "standard": {"overall": 71.45, "perAnswerType": {"other": 62.5, "number": 53.35, "yes/no": 86.5}}, "team_name_order": 10, "team_name": "EnmiaoFeng", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 75.26, "perAnswerType": {"other": 65.91, "number": 58.91, "yes/no": 90.33}}, "dev": {"overall": 75.0, "perAnswerType": {"other": 65.69, "number": 59.2, "yes/no": 90.09}}, "standard": {"overall": 75.23, "perAnswerType": {"other": 65.75, "number": 59.17, "yes/no": 90.36}}, "team_name_order": 11, "team_name": "MIL@HDU", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 72.03, "perAnswerType": {"other": 62.21, "number": 54.49, "yes/no": 87.95}}, "dev": {"overall": 71.84, "perAnswerType": {"other": 61.94, "number": 54.82, "yes/no": 87.93}}, "standard": {"overall": 72.08, "perAnswerType": {"other": 62.11, "number": 54.15, "yes/no": 88.24}}, "team_name_order": 12, "team_name": "HappyTeam", "ref": "", "method": ""}, {"team_members": "Anonymous (EMNLP submission)", "challenge": {"overall": 74.38, "perAnswerType": {"other": 65.32, "number": 57.29, "yes/no": 89.33}}, "dev": {"overall": 74.16, "perAnswerType": {"other": 65.14, "number": 56.85, "yes/no": 89.31}}, "standard": {"overall": 74.34, "perAnswerType": {"other": 65.22, "number": 56.69, "yes/no": 89.45}}, "team_name_order": 13, "team_name": "LXRT", "ref": "", "method": "Cross-Modality Transformer; Pre-trained from scratch; Fine-tuned on VQA."}, {"team_members": "Linjie Li, Zhe Gan, Yu Cheng, Faisal Ahmed, Ahmed El Kholy, Sarah Panda, Jingjing Liu (Microsoft Dynamics 365 AI)", "challenge": {"overall": 72.59, "perAnswerType": {"other": 63.08, "number": 56.23, "yes/no": 87.87}}, "dev": {"overall": 72.13, "perAnswerType": {"other": 62.64, "number": 55.71, "yes/no": 87.57}}, "standard": {"overall": 72.48, "perAnswerType": {"other": 62.85, "number": 55.39, "yes/no": 88.04}}, "team_name_order": 14, "team_name": "MS D365 AI", "ref": "https://arxiv.org/abs/1903.12314", "method": "We propose a Relation-aware Graph Attention Network (ReGAT), which encodes each image into a graph and models multi-type inter-object relations via a graph attention mechanism, to learn question-adaptive relation representations. Two types of visual object relations are explored: (i) Explicit Relations that represent geometric positions and semantic interactions between objects; and (ii) Implicit Relations that capture the hidden dynamics between image regions."}, {"team_members": "", "challenge": {"overall": 75.01, "perAnswerType": {"other": 65.89, "number": 59.01, "yes/no": 89.74}}, "dev": {"overall": 74.71, "perAnswerType": {"other": 65.39, "number": 58.89, "yes/no": 89.81}}, "standard": {"overall": 74.89, "perAnswerType": {"other": 65.69, "number": 58.36, "yes/no": 89.81}}, "team_name_order": 15, "team_name": "MSM@MSRA", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 69.87, "perAnswerType": {"other": 60.12, "number": 52.26, "yes/no": 85.74}}, "dev": {"overall": 69.7, "perAnswerType": {"other": 59.97, "number": 52.49, "yes/no": 85.64}}, "standard": {"overall": 69.97, "perAnswerType": {"other": 60.14, "number": 52.16, "yes/no": 85.95}}, "team_name_order": 16, "team_name": "Northwest_eagle", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 69.08, "perAnswerType": {"other": 59.94, "number": 47.72, "yes/no": 85.28}}, "dev": {"overall": 68.82, "perAnswerType": {"other": 59.76, "number": 47.35, "yes/no": 85.18}}, "standard": {"overall": 68.99, "perAnswerType": {"other": 59.91, "number": 46.98, "yes/no": 85.22}}, "team_name_order": 17, "team_name": "PL-UESTC", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 69.69, "perAnswerType": {"other": 60.58, "number": 49.12, "yes/no": 85.62}}, "dev": {"overall": 69.42, "perAnswerType": {"other": 60.3, "number": 48.48, "yes/no": 85.69}}, "standard": {"overall": 69.71, "perAnswerType": {"other": 60.44, "number": 48.47, "yes/no": 85.93}}, "team_name_order": 18, "team_name": "SYSU-VQA-Jokie", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 68.48, "perAnswerType": {"other": 59.5, "number": 47.65, "yes/no": 84.33}}, "dev": {"overall": 68.2, "perAnswerType": {"other": 59.25, "number": 47.63, "yes/no": 84.17}}, "standard": {"overall": 68.57, "perAnswerType": {"other": 59.49, "number": 47.5, "yes/no": 84.55}}, "team_name_order": 19, "team_name": "SZTSUH", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 72.84, "perAnswerType": {"other": 63.87, "number": 54.67, "yes/no": 87.99}}, "dev": {"overall": 72.51, "perAnswerType": {"other": 63.42, "number": 54.39, "yes/no": 87.97}}, "standard": {"overall": 72.74, "perAnswerType": {"other": 63.51, "number": 54.04, "yes/no": 88.26}}, "team_name_order": 20, "team_name": "THEQS", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 68.01, "perAnswerType": {"other": 58.85, "number": 46.6, "yes/no": 84.23}}, "dev": {"overall": 67.71, "perAnswerType": {"other": 58.49, "number": 46.8, "yes/no": 84.09}}, "standard": {"overall": 68.08, "perAnswerType": {"other": 58.6, "number": 46.42, "yes/no": 84.67}}, "team_name_order": 21, "team_name": "TeesriAankh", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 70.21, "perAnswerType": {"other": 60.63, "number": 53.91, "yes/no": 85.53}}, "dev": {"overall": 69.89, "perAnswerType": {"other": 60.22, "number": 53.8, "yes/no": 85.46}}, "standard": {"overall": 70.32, "perAnswerType": {"other": 60.52, "number": 53.56, "yes/no": 85.99}}, "team_name_order": 22, "team_name": "VQA_LTRC", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 73.35, "perAnswerType": {"other": 64.3, "number": 57.87, "yes/no": 87.86}}, "dev": {"overall": 73.03, "perAnswerType": {"other": 63.67, "number": 58.1, "yes/no": 87.92}}, "standard": {"overall": 73.13, "perAnswerType": {"other": 63.92, "number": 57.14, "yes/no": 87.91}}, "team_name_order": 23, "team_name": "XFZ", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 69.78, "perAnswerType": {"other": 59.96, "number": 51.82, "yes/no": 85.82}}, "dev": {"overall": 69.52, "perAnswerType": {"other": 59.73, "number": 51.38, "yes/no": 85.8}}, "standard": {"overall": 69.86, "perAnswerType": {"other": 60.01, "number": 51.84, "yes/no": 85.92}}, "team_name_order": 24, "team_name": "YAU_VQA_TEAM", "ref": "", "method": ""}, {"team_members": "woosik yang ( Korea univ.)", "challenge": {"overall": 64.11, "perAnswerType": {"other": 54.49, "number": 42.75, "yes/no": 80.85}}, "dev": {"overall": 63.94, "perAnswerType": {"other": 54.2, "number": 42.81, "yes/no": 80.99}}, "standard": {"overall": 64.3, "perAnswerType": {"other": 54.48, "number": 42.94, "yes/no": 81.2}}, "team_name_order": 25, "team_name": "casablanca", "ref": "", "method": "added global image information "}, {"team_members": "Liyang Zhang (UESTC)", "challenge": {"overall": 70.38, "perAnswerType": {"other": 61.12, "number": 51.21, "yes/no": 86.12}}, "dev": {"overall": 69.98, "perAnswerType": {"other": 60.74, "number": 50.92, "yes/no": 85.86}}, "standard": {"overall": 70.19, "perAnswerType": {"other": 60.87, "number": 50.79, "yes/no": 85.99}}, "team_name_order": 26, "team_name": "cfm_leon", "ref": "", "method": "We add the other visual input extracted from label of images, and fuse two visual input into one. Also, we use the attention network based on bilinear attention. We ensemble about 8 model to get better results."}, {"team_members": "", "challenge": {"overall": 70.41, "perAnswerType": {"other": 61.47, "number": 49.31, "yes/no": 86.3}}, "dev": {"overall": 70.01, "perAnswerType": {"other": 61.06, "number": 48.97, "yes/no": 86.12}}, "standard": {"overall": 70.24, "perAnswerType": {"other": 61.18, "number": 48.46, "yes/no": 86.37}}, "team_name_order": 27, "team_name": "channelcs", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 71.62, "perAnswerType": {"other": 62.7, "number": 54.47, "yes/no": 86.44}}, "dev": {"overall": 71.26, "perAnswerType": {"other": 62.37, "number": 54.41, "yes/no": 86.15}}, "standard": {"overall": 71.62, "perAnswerType": {"other": 62.56, "number": 54.51, "yes/no": 86.52}}, "team_name_order": 28, "team_name": "fm", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 62.83, "perAnswerType": {"other": 53.71, "number": 39.37, "yes/no": 79.57}}, "dev": {"overall": 62.62, "perAnswerType": {"other": 53.58, "number": 39.52, "yes/no": 79.4}}, "standard": {"overall": 62.96, "perAnswerType": {"other": 53.92, "number": 39.43, "yes/no": 79.54}}, "team_name_order": 29, "team_name": "fufu", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 71.39, "perAnswerType": {"other": 62.52, "number": 53.6, "yes/no": 86.32}}, "dev": {"overall": 71.15, "perAnswerType": {"other": 62.24, "number": 53.91, "yes/no": 86.17}}, "standard": {"overall": 71.48, "perAnswerType": {"other": 62.47, "number": 53.6, "yes/no": 86.52}}, "team_name_order": 30, "team_name": "hust_vqateam", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 63.16, "perAnswerType": {"other": 53.76, "number": 41.29, "yes/no": 79.78}}, "dev": {"overall": 62.69, "perAnswerType": {"other": 53.26, "number": 40.89, "yes/no": 79.55}}, "standard": {"overall": 63.15, "perAnswerType": {"other": 53.73, "number": 40.54, "yes/no": 79.91}}, "team_name_order": 31, "team_name": "kodingcoding", "ref": "", "method": ""}, {"team_members": "Hongli Ding, Wei Mei, Liang Huang, Huanian Zhan", "challenge": {"overall": 72.94, "perAnswerType": {"other": 63.97, "number": 55.17, "yes/no": 87.97}}, "dev": {"overall": 72.8, "perAnswerType": {"other": 63.82, "number": 55.53, "yes/no": 87.89}}, "standard": {"overall": 72.92, "perAnswerType": {"other": 63.8, "number": 55.36, "yes/no": 88.02}}, "team_name_order": 32, "team_name": "ks_vqa", "ref": "", "method": "we modified the model of pythia with adding additional word embedding infos, and ensemble 7 models"}, {"team_members": "", "challenge": {"overall": 68.23, "perAnswerType": {"other": 58.83, "number": 47.12, "yes/no": 84.65}}, "dev": {"overall": 67.86, "perAnswerType": {"other": 58.39, "number": 47.31, "yes/no": 84.44}}, "standard": {"overall": 68.16, "perAnswerType": {"other": 58.67, "number": 46.6, "yes/no": 84.73}}, "team_name_order": 33, "team_name": "memo", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 70.19, "perAnswerType": {"other": 60.49, "number": 53.64, "yes/no": 85.72}}, "dev": {"overall": 69.82, "perAnswerType": {"other": 60.1, "number": 53.53, "yes/no": 85.5}}, "standard": {"overall": 70.13, "perAnswerType": {"other": 60.41, "number": 53.16, "yes/no": 85.76}}, "team_name_order": 34, "team_name": "personal", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 68.83, "perAnswerType": {"other": 59.84, "number": 51.67, "yes/no": 83.72}}, "dev": {"overall": 68.57, "perAnswerType": {"other": 59.63, "number": 51.59, "yes/no": 83.54}}, "standard": {"overall": 68.87, "perAnswerType": {"other": 59.65, "number": 51.55, "yes/no": 84.02}}, "team_name_order": 35, "team_name": "tsfy", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 71.62, "perAnswerType": {"other": 62.7, "number": 54.2, "yes/no": 86.5}}, "dev": {"overall": 71.28, "perAnswerType": {"other": 62.42, "number": 54.19, "yes/no": 86.19}}, "standard": {"overall": 71.66, "perAnswerType": {"other": 62.6, "number": 54.25, "yes/no": 86.65}}, "team_name_order": 36, "team_name": "vqa_team_tsuh", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 71.39, "perAnswerType": {"other": 62.49, "number": 53.52, "yes/no": 86.35}}, "dev": {"overall": 71.13, "perAnswerType": {"other": 62.28, "number": 53.57, "yes/no": 86.16}}, "standard": {"overall": 71.48, "perAnswerType": {"other": 62.51, "number": 53.41, "yes/no": 86.53}}, "team_name_order": 37, "team_name": "vteam", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 69.32, "perAnswerType": {"other": 60.47, "number": 48.06, "yes/no": 85.16}}, "dev": {"overall": 68.98, "perAnswerType": {"other": 60.04, "number": 48.08, "yes/no": 85.05}}, "standard": {"overall": 69.28, "perAnswerType": {"other": 60.16, "number": 47.27, "yes/no": 85.55}}, "team_name_order": 38, "team_name": "ywt-test", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 70.96, "perAnswerType": {"other": 61.98, "number": 52.57, "yes/no": 86.17}}, "dev": {"overall": 70.54, "perAnswerType": {"other": 61.65, "number": 52.0, "yes/no": 85.88}}, "standard": {"overall": 71.01, "perAnswerType": {"other": 61.98, "number": 52.11, "yes/no": 86.36}}, "team_name_order": 39, "team_name": "zhanghz", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 61.94, "perAnswerType": {"other": 51.25, "number": 42.56, "yes/no": 79.37}}, "dev": {"overall": 61.7, "perAnswerType": {"other": 51.05, "number": 42.05, "yes/no": 79.38}}, "standard": {"overall": 62.11, "perAnswerType": {"other": 51.4, "number": 41.75, "yes/no": 79.77}}, "team_name_order": 40, "team_name": "zrc", "ref": "", "method": ""}, {"team_members": "", "challenge": {"overall": 71.37, "perAnswerType": {"other": 62.49, "number": 53.66, "yes/no": 86.29}}, "dev": {"overall": 71.12, "perAnswerType": {"other": 62.28, "number": 53.61, "yes/no": 86.13}}, "standard": {"overall": 71.47, "perAnswerType": {"other": 62.43, "number": 53.46, "yes/no": 86.6}}, "team_name_order": 41, "team_name": "zwlab", "ref": "", "method": ""}, {"date": "2019-06-05"}]