Closing the convergence gap of SGD without replacement Shashank Rajput, Anant Gupta, Dimitris Papailiopoulos
<latexit sha1_base64="bFz0n2P5KY7YuTtmLx+ZHinFcPo=">ACKnicdVDLihNBFK0eX2N8RV26uZgILkJT3Y6ZcTeDG5cjmJmBJDS3q29PiqlHU1UthCbf48ZfcTMLZXDrh1jJRFDRAwWHc+7h1j1lo6QPnF8lOzdu3rp9Z/du7979Bw8f9R8/OfG2dYImwirzkr0pKShSZB0VnjCHWp6LS8eLv2Tz+S89KaD2HZ0FzjuZG1FBiVPSPjgIQigXIQG6jcCjbhTBsC7kEByaymq1hNpZDcNZVxfZaFbZ4Ed1YWarYdEf8JS/zrM9DjzNx2/yg3Ek41f7Gc8hS/kGA7bFcdG/jHRajJBKPR+mvEmzDt0QpFq96s9dSguMBzmkZqUJOfd5tTV/AiKhXU1sVnAmzU3xMdau+XuoyTGsPC/+2txX950zbUB/NOmqYNZMT1orpVECyse4NKOhIh9lBJFE7Gv4JYoEMRe/O9WMKvS+H/5CRPM5m7/PB4d62jl32jD1nL1nG9tkhe8eO2YQJ9ol9YV/Zt+RzcplcJd+vR3eSbeYp+wPJj5+oTqYn</latexit> <latexit sha1_base64="bFz0n2P5KY7YuTtmLx+ZHinFcPo=">ACKnicdVDLihNBFK0eX2N8RV26uZgILkJT3Y6ZcTeDG5cjmJmBJDS3q29PiqlHU1UthCbf48ZfcTMLZXDrh1jJRFDRAwWHc+7h1j1lo6QPnF8lOzdu3rp9Z/du7979Bw8f9R8/OfG2dYImwirzkr0pKShSZB0VnjCHWp6LS8eLv2Tz+S89KaD2HZ0FzjuZG1FBiVPSPjgIQigXIQG6jcCjbhTBsC7kEByaymq1hNpZDcNZVxfZaFbZ4Ed1YWarYdEf8JS/zrM9DjzNx2/yg3Ek41f7Gc8hS/kGA7bFcdG/jHRajJBKPR+mvEmzDt0QpFq96s9dSguMBzmkZqUJOfd5tTV/AiKhXU1sVnAmzU3xMdau+XuoyTGsPC/+2txX950zbUB/NOmqYNZMT1orpVECyse4NKOhIh9lBJFE7Gv4JYoEMRe/O9WMKvS+H/5CRPM5m7/PB4d62jl32jD1nL1nG9tkhe8eO2YQJ9ol9YV/Zt+RzcplcJd+vR3eSbeYp+wPJj5+oTqYn</latexit> <latexit sha1_base64="bFz0n2P5KY7YuTtmLx+ZHinFcPo=">ACKnicdVDLihNBFK0eX2N8RV26uZgILkJT3Y6ZcTeDG5cjmJmBJDS3q29PiqlHU1UthCbf48ZfcTMLZXDrh1jJRFDRAwWHc+7h1j1lo6QPnF8lOzdu3rp9Z/du7979Bw8f9R8/OfG2dYImwirzkr0pKShSZB0VnjCHWp6LS8eLv2Tz+S89KaD2HZ0FzjuZG1FBiVPSPjgIQigXIQG6jcCjbhTBsC7kEByaymq1hNpZDcNZVxfZaFbZ4Ed1YWarYdEf8JS/zrM9DjzNx2/yg3Ek41f7Gc8hS/kGA7bFcdG/jHRajJBKPR+mvEmzDt0QpFq96s9dSguMBzmkZqUJOfd5tTV/AiKhXU1sVnAmzU3xMdau+XuoyTGsPC/+2txX950zbUB/NOmqYNZMT1orpVECyse4NKOhIh9lBJFE7Gv4JYoEMRe/O9WMKvS+H/5CRPM5m7/PB4d62jl32jD1nL1nG9tkhe8eO2YQJ9ol9YV/Zt+RzcplcJd+vR3eSbeYp+wPJj5+oTqYn</latexit> <latexit sha1_base64="bFz0n2P5KY7YuTtmLx+ZHinFcPo=">ACKnicdVDLihNBFK0eX2N8RV26uZgILkJT3Y6ZcTeDG5cjmJmBJDS3q29PiqlHU1UthCbf48ZfcTMLZXDrh1jJRFDRAwWHc+7h1j1lo6QPnF8lOzdu3rp9Z/du7979Bw8f9R8/OfG2dYImwirzkr0pKShSZB0VnjCHWp6LS8eLv2Tz+S89KaD2HZ0FzjuZG1FBiVPSPjgIQigXIQG6jcCjbhTBsC7kEByaymq1hNpZDcNZVxfZaFbZ4Ed1YWarYdEf8JS/zrM9DjzNx2/yg3Ek41f7Gc8hS/kGA7bFcdG/jHRajJBKPR+mvEmzDt0QpFq96s9dSguMBzmkZqUJOfd5tTV/AiKhXU1sVnAmzU3xMdau+XuoyTGsPC/+2txX950zbUB/NOmqYNZMT1orpVECyse4NKOhIh9lBJFE7Gv4JYoEMRe/O9WMKvS+H/5CRPM5m7/PB4d62jl32jD1nL1nG9tkhe8eO2YQJ9ol9YV/Zt+RzcplcJd+vR3eSbeYp+wPJj5+oTqYn</latexit> <latexit sha1_base64="3fen9BguvaKkANsAF6iVMK8pUE=">ACMHicbZDfaxNBEMfnqtWaVo0/3nwZDEJFDXciKIQ6IN9rGDaQhLOuc1cs3Rv79idk8Qjf5Iv/in6UkEpvpXuEmK2NYvLHz5zAyz80qo73E8fdo7crV9WvXN260Nrdu3rdvnN35e1U9xXpSndYUaejbcFy2GDyvHVGSGD7LjnUX94CM7r0v7XmYVjwo6sjrXiSgtP12mjbyJnj6zfTVPAZDslUE8KhpcwQ5qneDvzx079ceCoNao8yYfTCFXr9iedpuxN346XwsknOTKd3P8/AMBe2v46HJeqLtiKMuT9IkrGTXkRCvD89aw9lyROqYjHgRrqWA/apYHz/FRIGPMSxeFVzSfycaKryfFVnoLEgm/mJtAf9XG9Svxo12la1sFWrRXltUEpcpIdj7ViJmQVDyunwV1QTcqQkZNwKISQXT75s9p93k7ibvEs6vRew0gY8gIewDQm8hB7swh70QcFn+AY/4Gf0JTqJTqNfq9a16GzmHpxT9PsPibipjw=</latexit> <latexit sha1_base64="Kow3Z1P1X7aZhSDItpuefcsQw2I=">ACMHicbZDfSxtBEMf3tLa1pa3/oyNBQU23BXChVBCPhQHy0GkjCMbeZM4t7e8fuXEk8if54p9SXxRaiq/9K7r5QWm1X1j48pkZuebFo5DsPbYGn50crjJ6trtafP1p9v1F9snri8tJLaMte57SToSCtDbVasqVNYwizRdJqcH07rp1/JOpWbLzwuqJ/hmVGpksgexfVPo7ji3WgC+wejmOEd9FAXQ4SewUQjpLHa9nzn7R/ONOIKlAMeEjimApy6oElcb4TNcCZ4aKFabS20pmO4/q3iCXZUaGpUbnulFYcL9Cy0pqmtR6paMC5TmeUdbgxm5fjU7eAJvPBlAmlv/DMOM/j1RYebcOEt8Z4Y8dPdrU/i/WrfkdK9fKVOUTEbOF6WlBs5hmh4MlCXJeuwNSqv8X0EO0aJkn3HNhxDdP/mhOXnfjMJm9DlqtD6IuVbFK/FabItIfBQtcSORVtIcSmuxXfxI7gKboKfwd28dSlYzLwU/yj49RsqsqrH</latexit> <latexit sha1_base64="Kow3Z1P1X7aZhSDItpuefcsQw2I=">ACMHicbZDfSxtBEMf3tLa1pa3/oyNBQU23BXChVBCPhQHy0GkjCMbeZM4t7e8fuXEk8if54p9SXxRaiq/9K7r5QWm1X1j48pkZuebFo5DsPbYGn50crjJ6trtafP1p9v1F9snri8tJLaMte57SToSCtDbVasqVNYwizRdJqcH07rp1/JOpWbLzwuqJ/hmVGpksgexfVPo7ji3WgC+wejmOEd9FAXQ4SewUQjpLHa9nzn7R/ONOIKlAMeEjimApy6oElcb4TNcCZ4aKFabS20pmO4/q3iCXZUaGpUbnulFYcL9Cy0pqmtR6paMC5TmeUdbgxm5fjU7eAJvPBlAmlv/DMOM/j1RYebcOEt8Z4Y8dPdrU/i/WrfkdK9fKVOUTEbOF6WlBs5hmh4MlCXJeuwNSqv8X0EO0aJkn3HNhxDdP/mhOXnfjMJm9DlqtD6IuVbFK/FabItIfBQtcSORVtIcSmuxXfxI7gKboKfwd28dSlYzLwU/yj49RsqsqrH</latexit> <latexit sha1_base64="4Npo/c1Rtmo0CPzKk2Fd+1l0KI0=">ACMHicbZBRaxNBEMf3WltjWtuoj74MhkJLa7grgiIAR/0sYJpC0k45jZzZK9vWN3ThKPfKS+9KPUF4UW8dVP4SYNokn/sPDnNzPMzj8ptHIchj+CtfUHG5sPa4/qW9uPd3YbT56eury0kjoy17k9T9CRVoY6rFjTeWEJs0TWTJ6P6ufSHrVG4+86SgfoYXRqVKInsUNz6M4oPoym8fTeOGV5CD3UxROgZTDRCGqt9zw+O/nKmMVegHPCQwDEV4NRXmsaNZtgK54JVEy1MUyx0Ejeue4NclhkZlhqd60Zhwf0KLSupaVrvlY4KlCO8oK63BjNy/Wp+8BT2PBlAmlv/DMOc/jtRYebcJEt8Z4Y8dMu1Gbyv1i05fdOvlClKJiPvFqWlBs5hlh4MlCXJeuINSqv8X0EO0aJkn3HdhxAtn7xqTo9bUdiKPkXN9qtFHDXxXLwQ+yISr0VbfBQnoiOkuBTfxI24Da6C78HP4Nd61qwmHkm/lPw+w+DLKgP</latexit> <latexit sha1_base64="Mwv+2sFCgxVJoPAQ2pdYIYCMGM=">ACLHicbVDLSgMxFM34rPVdekmWIW6KTNulEqhUBCXFWwrdOqQSTM2NMkMSUZawnyQG39FEBcWcet3mD4Waj0QOJxzLjf3hAmjSrvu2FlaXldW89t5De3tnd2C3v7LRWnEpMmjlks70KkCKOCNDXVjNwlkiAeMtIOB/WJ34kUtFY3OpRQrocPQgaUYy0lYJCvSFjm+WwAo/9VPRslGgzIzPqcj8y6vS8LRS9SOJsPEyYyWV8sDQqpfdiyig1j4OCkW37E4BF4k3J0UwRyMovPq9GKecCI0ZUqrjuYnuGiQ1xYxkeT9VJEF4gB5Ix1KBOFdMz02gydW6cEolvYJDafqzwmDuFIjHtokR7qv/noT8T+vk+romuoSFJNBJ4tilIGdQwnzcEelQRrNrIEYUntXyHuI1uMtqXlbQne35MXSeus7Ll7+asWLuY15EDh+AIlIAHzkENXIMGaAIMnsALeAdj59l5cz6cz1l0yZnPHIBfcL6+AdJKp+Q=</latexit> <latexit sha1_base64="Mwv+2sFCgxVJoPAQ2pdYIYCMGM=">ACLHicbVDLSgMxFM34rPVdekmWIW6KTNulEqhUBCXFWwrdOqQSTM2NMkMSUZawnyQG39FEBcWcet3mD4Waj0QOJxzLjf3hAmjSrvu2FlaXldW89t5De3tnd2C3v7LRWnEpMmjlks70KkCKOCNDXVjNwlkiAeMtIOB/WJ34kUtFY3OpRQrocPQgaUYy0lYJCvSFjm+WwAo/9VPRslGgzIzPqcj8y6vS8LRS9SOJsPEyYyWV8sDQqpfdiyig1j4OCkW37E4BF4k3J0UwRyMovPq9GKecCI0ZUqrjuYnuGiQ1xYxkeT9VJEF4gB5Ix1KBOFdMz02gydW6cEolvYJDafqzwmDuFIjHtokR7qv/noT8T+vk+romuoSFJNBJ4tilIGdQwnzcEelQRrNrIEYUntXyHuI1uMtqXlbQne35MXSeus7Ll7+asWLuY15EDh+AIlIAHzkENXIMGaAIMnsALeAdj59l5cz6cz1l0yZnPHIBfcL6+AdJKp+Q=</latexit> <latexit sha1_base64="Mwv+2sFCgxVJoPAQ2pdYIYCMGM=">ACLHicbVDLSgMxFM34rPVdekmWIW6KTNulEqhUBCXFWwrdOqQSTM2NMkMSUZawnyQG39FEBcWcet3mD4Waj0QOJxzLjf3hAmjSrvu2FlaXldW89t5De3tnd2C3v7LRWnEpMmjlks70KkCKOCNDXVjNwlkiAeMtIOB/WJ34kUtFY3OpRQrocPQgaUYy0lYJCvSFjm+WwAo/9VPRslGgzIzPqcj8y6vS8LRS9SOJsPEyYyWV8sDQqpfdiyig1j4OCkW37E4BF4k3J0UwRyMovPq9GKecCI0ZUqrjuYnuGiQ1xYxkeT9VJEF4gB5Ix1KBOFdMz02gydW6cEolvYJDafqzwmDuFIjHtokR7qv/noT8T+vk+romuoSFJNBJ4tilIGdQwnzcEelQRrNrIEYUntXyHuI1uMtqXlbQne35MXSeus7Ll7+asWLuY15EDh+AIlIAHzkENXIMGaAIMnsALeAdj59l5cz6cz1l0yZnPHIBfcL6+AdJKp+Q=</latexit> <latexit sha1_base64="Mwv+2sFCgxVJoPAQ2pdYIYCMGM=">ACLHicbVDLSgMxFM34rPVdekmWIW6KTNulEqhUBCXFWwrdOqQSTM2NMkMSUZawnyQG39FEBcWcet3mD4Waj0QOJxzLjf3hAmjSrvu2FlaXldW89t5De3tnd2C3v7LRWnEpMmjlks70KkCKOCNDXVjNwlkiAeMtIOB/WJ34kUtFY3OpRQrocPQgaUYy0lYJCvSFjm+WwAo/9VPRslGgzIzPqcj8y6vS8LRS9SOJsPEyYyWV8sDQqpfdiyig1j4OCkW37E4BF4k3J0UwRyMovPq9GKecCI0ZUqrjuYnuGiQ1xYxkeT9VJEF4gB5Ix1KBOFdMz02gydW6cEolvYJDafqzwmDuFIjHtokR7qv/noT8T+vk+romuoSFJNBJ4tilIGdQwnzcEelQRrNrIEYUntXyHuI1uMtqXlbQne35MXSeus7Ll7+asWLuY15EDh+AIlIAHzkENXIMGaAIMnsALeAdj59l5cz6cz1l0yZnPHIBfcL6+AdJKp+Q=</latexit> Stochastic Gradient Descent (SGD) P n • F ( x ) := 1 Sampling with replacement Problem : min i =1 f i ( x ) n x • Algorithm: 1. At each iteration, sample f i randomly from { f 1 , . . . , f n } 2. x t +1 := x t � α r f i ( x t ) , α is the step size 3. Repeat for T iterations • SGD with replacement is theoretically well understood However, in practice we sample without replacement
Recommend
More recommend