Machine Learning Kernels

Brute Force Nearest-Neighbor Kernel

---

\(X \sim U[0, 1]\)

\(\epsilon \sim N(0, 1/3)\)

\(Y = \sin(4X) + \epsilon\)

# X ~ U[0, 1]
X <- runif(100, min = 0, max = 1)
# eps ~ N(0, 1/3)
eps <- rnorm(100, mean = 0, sd = sqrt(1/3))
  
# Y random variable
f <- function(X, eps) {
  sin(4*X) + eps
}

Y <- f(X, eps)
plot(X, Y, pch=19, col="blue")

\(f_{knn}\) Approximation to \(f\) with Nearest-Neighbor Running-Mean Smoother

# Example elementwise substraction
# > X <- c(1, 2, 3, 4)
# > x <- 2
# > X - x
distances <- function(x, X) {
  return(abs(X - x))
}
# Example for sort index.return
# > u <- c(3, 5, 2, 7, 1, 8)
# > sort(u, index.return = TRUE)$ix
# [1] 5 3 1 2 4 6
# > sort(u, index.return = TRUE)$ix[1:3]
# [1] 5 3 1
k_smallest_elements <- function(d, k) {
  return(sort(d, index.return = TRUE)$ix[1:k])
}
# kNN 
k_nearest_neighbors_indices <- function(x, k, X) {
  return(k_smallest_elements(distances(x, X), k))
}
f_knn <- function(x, k, X, Y) {
  Y_knn <- numeric(length=length(x))
  for (i in 1:length(x)) {
    knn_idxs <- k_nearest_neighbors_indices(x[i], k, X)
    Y_knn[i] <- mean(Y[knn_idxs])
  }
  return(Y_knn)
}
x <- seq(0, 1, by = 0.01)
k <- 30
plot(X, Y, col="blue", pch=19)
lines(x, f_knn(x, k, X, Y), col="green", lwd=2)