diff --git a/src/Avalonia.Base/Input/GestureRecognizers/ScrollGestureRecognizer.cs b/src/Avalonia.Base/Input/GestureRecognizers/ScrollGestureRecognizer.cs
index daffe0ce77f..7bf5bc51a2d 100644
--- a/src/Avalonia.Base/Input/GestureRecognizers/ScrollGestureRecognizer.cs
+++ b/src/Avalonia.Base/Input/GestureRecognizers/ScrollGestureRecognizer.cs
@@ -102,6 +102,8 @@ protected override void PointerPressed(PointerPressedEventArgs e)
                 _gestureId = ScrollGestureEventArgs.GetNextFreeId();
                 _rootTarget = (Visual?)(Target as Visual)?.VisualRoot;
                 _trackedRootPoint = _pointerPressedPoint = e.GetPosition(_rootTarget);
+                _velocityTracker = new VelocityTracker();
+                _velocityTracker?.AddPosition(TimeSpan.FromMilliseconds(e.Timestamp), default);
             }
         }
 
@@ -117,9 +119,7 @@ protected override void PointerMoved(PointerEventArgs e)
                     if (CanVerticallyScroll && Math.Abs(_trackedRootPoint.Y - rootPoint.Y) > ScrollStartDistance)
                         _scrolling = true;
                     if (_scrolling)
-                    {
-                        _velocityTracker = new VelocityTracker();
-                        
+                    {                        
                         // Correct _trackedRootPoint with ScrollStartDistance, so scrolling does not start with a skip of ScrollStartDistance
                         _trackedRootPoint = new Point(
                             _trackedRootPoint.X - (_trackedRootPoint.X >= rootPoint.X ? ScrollStartDistance : -ScrollStartDistance),
diff --git a/src/Avalonia.Base/Input/GestureRecognizers/VelocityTracker.cs b/src/Avalonia.Base/Input/GestureRecognizers/VelocityTracker.cs
index 77ebfeee3a9..4e46e19d026 100644
--- a/src/Avalonia.Base/Input/GestureRecognizers/VelocityTracker.cs
+++ b/src/Avalonia.Base/Input/GestureRecognizers/VelocityTracker.cs
@@ -139,6 +139,9 @@ public void AddPosition(TimeSpan time, Vector position)
                 sampleCount++;
             } while (sampleCount < HistorySize);
 
+            var offset = newestSample.Point - oldestSample.Point;
+            var duration = newestSample.Time - oldestSample.Time;
+
             if (sampleCount >= MinSampleSize)
             {
                 var xFit = LeastSquaresSolver.Solve(2, time.Slice(0, sampleCount), x.Slice(0, sampleCount), w.Slice(0, sampleCount));
@@ -150,20 +153,31 @@ public void AddPosition(TimeSpan time, Vector position)
                         return new VelocityEstimate( // convert from pixels/ms to pixels/s
                           PixelsPerSecond: new Vector(xFit.Coefficients[1] * 1000, yFit.Coefficients[1] * 1000),
                           Confidence: xFit.Confidence * yFit.Confidence,
-                          Duration: newestSample.Time - oldestSample.Time,
-                          Offset: newestSample.Point - oldestSample.Point
+                          Duration: duration,
+                          Offset: offset
                         );
                     }
                 }
             }
+            else if(sampleCount > 1)
+            {
+                // Return linear velocity if we don't have enough samples
+                var distance = newestSample.Point - oldestSample.Point;
+                return new VelocityEstimate(
+                  PixelsPerSecond: new Vector(distance.X / duration.Milliseconds * 1000, distance.Y / duration.Milliseconds * 1000),
+                  Confidence: 1,
+                  Duration: duration,
+                  Offset: offset
+                );
+            }
 
             // We're unable to make a velocity estimate but we did have at least one
             // valid pointer position.
             return new VelocityEstimate(
               PixelsPerSecond: Vector.Zero,
               Confidence: 1.0,
-              Duration: newestSample.Time - oldestSample.Time,
-              Offset: newestSample.Point - oldestSample.Point
+              Duration: duration,
+              Offset: offset
             );
         }